knight/datahub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
datahub/docs/backend/localcsv.html

384 lines
25 KiB
HTML
Raw Blame History

<!DOCTYPE html>
<html>
<head>
<meta http-equiv="content-type" content="text/html;charset=utf-8">
<title>localcsv.js</title>
<link rel="stylesheet" href="pycco.css">
</head>
<body>
<div id='container'>
<div id="background"></div>
<div class='section'>
<div class='docs'><h1>localcsv.js</h1></div>
</div>
<div class='clearall'>
<div class='section' id='section-0'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-0'>#</a>
</div>
</div>
<div class='code'>
<div class="highlight"><pre><span class="k">this</span><span class="p">.</span><span class="nx">recline</span> <span class="o">=</span> <span class="k">this</span><span class="p">.</span><span class="nx">recline</span> <span class="o">||</span> <span class="p">{};</span>
<span class="k">this</span><span class="p">.</span><span class="nx">recline</span><span class="p">.</span><span class="nx">Backend</span> <span class="o">=</span> <span class="k">this</span><span class="p">.</span><span class="nx">recline</span><span class="p">.</span><span class="nx">Backend</span> <span class="o">||</span> <span class="p">{};</span>
<span class="p">(</span><span class="kd">function</span><span class="p">(</span><span class="nx">$</span><span class="p">,</span> <span class="nx">my</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">my</span><span class="p">.</span><span class="nx">loadFromCSVFile</span> <span class="o">=</span> <span class="kd">function</span><span class="p">(</span><span class="nx">file</span><span class="p">,</span> <span class="nx">callback</span><span class="p">)</span> <span class="p">{</span>
<span class="kd">var</span> <span class="nx">metadata</span> <span class="o">=</span> <span class="p">{</span>
<span class="nx">id</span><span class="o">:</span> <span class="nx">file</span><span class="p">.</span><span class="nx">name</span><span class="p">,</span>
<span class="nx">file</span><span class="o">:</span> <span class="nx">file</span>
<span class="p">};</span>
<span class="kd">var</span> <span class="nx">reader</span> <span class="o">=</span> <span class="k">new</span> <span class="nx">FileReader</span><span class="p">();</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-1'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-1'>#</a>
</div>
<p>TODO</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">reader</span><span class="p">.</span><span class="nx">onload</span> <span class="o">=</span> <span class="kd">function</span><span class="p">(</span><span class="nx">e</span><span class="p">)</span> <span class="p">{</span>
<span class="kd">var</span> <span class="nx">dataset</span> <span class="o">=</span> <span class="nx">my</span><span class="p">.</span><span class="nx">csvToDataset</span><span class="p">(</span><span class="nx">e</span><span class="p">.</span><span class="nx">target</span><span class="p">.</span><span class="nx">result</span><span class="p">);</span>
<span class="nx">callback</span><span class="p">(</span><span class="nx">dataset</span><span class="p">);</span>
<span class="p">};</span>
<span class="nx">reader</span><span class="p">.</span><span class="nx">onerror</span> <span class="o">=</span> <span class="kd">function</span> <span class="p">(</span><span class="nx">e</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">alert</span><span class="p">(</span><span class="s1">&#39;Failed to load file. Code: &#39;</span> <span class="o">+</span> <span class="nx">e</span><span class="p">.</span><span class="nx">target</span><span class="p">.</span><span class="nx">error</span><span class="p">.</span><span class="nx">code</span><span class="p">);</span>
<span class="p">};</span>
<span class="nx">reader</span><span class="p">.</span><span class="nx">readAsText</span><span class="p">(</span><span class="nx">file</span><span class="p">);</span>
<span class="p">};</span>
<span class="nx">my</span><span class="p">.</span><span class="nx">csvToDataset</span> <span class="o">=</span> <span class="kd">function</span><span class="p">(</span><span class="nx">csvString</span><span class="p">)</span> <span class="p">{</span>
<span class="kd">var</span> <span class="nx">out</span> <span class="o">=</span> <span class="nx">my</span><span class="p">.</span><span class="nx">parseCSV</span><span class="p">(</span><span class="nx">csvString</span><span class="p">);</span>
<span class="nx">fields</span> <span class="o">=</span> <span class="nx">_</span><span class="p">.</span><span class="nx">map</span><span class="p">(</span><span class="nx">out</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="kd">function</span><span class="p">(</span><span class="nx">cell</span><span class="p">)</span> <span class="p">{</span>
<span class="k">return</span> <span class="p">{</span> <span class="nx">id</span><span class="o">:</span> <span class="nx">cell</span><span class="p">,</span> <span class="nx">label</span><span class="o">:</span> <span class="nx">cell</span> <span class="p">};</span>
<span class="p">});</span>
<span class="kd">var</span> <span class="nx">data</span> <span class="o">=</span> <span class="nx">_</span><span class="p">.</span><span class="nx">map</span><span class="p">(</span><span class="nx">out</span><span class="p">.</span><span class="nx">slice</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="kd">function</span><span class="p">(</span><span class="nx">row</span><span class="p">)</span> <span class="p">{</span>
<span class="kd">var</span> <span class="nx">_doc</span> <span class="o">=</span> <span class="p">{};</span>
<span class="nx">_</span><span class="p">.</span><span class="nx">each</span><span class="p">(</span><span class="nx">out</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="kd">function</span><span class="p">(</span><span class="nx">fieldId</span><span class="p">,</span> <span class="nx">idx</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">_doc</span><span class="p">[</span><span class="nx">fieldId</span><span class="p">]</span> <span class="o">=</span> <span class="nx">row</span><span class="p">[</span><span class="nx">idx</span><span class="p">];</span>
<span class="p">});</span>
<span class="k">return</span> <span class="nx">_doc</span><span class="p">;</span>
<span class="p">});</span>
<span class="kd">var</span> <span class="nx">dataset</span> <span class="o">=</span> <span class="nx">recline</span><span class="p">.</span><span class="nx">Backend</span><span class="p">.</span><span class="nx">createDataset</span><span class="p">(</span><span class="nx">data</span><span class="p">,</span> <span class="nx">fields</span><span class="p">);</span>
<span class="k">return</span> <span class="nx">dataset</span><span class="p">;</span>
<span class="p">};</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-2'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-2'>#</a>
</div>
<p>Converts a Comma Separated Values string into an array of arrays.
Each line in the CSV becomes an array.</p>
<p>Empty fields are converted to nulls and non-quoted numbers are converted to integers or floats.</p>
<p>@return The CSV parsed as an array
@type Array</p>
<p>@param {String} s The string to convert
@param {Boolean} [trm=false] If set to True leading and trailing whitespace is stripped off of each non-quoted field as it is imported</p>
<p>Heavily based on uselesscode's JS CSV parser (MIT Licensed):
thttp://www.uselesscode.org/javascript/csv/</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">my</span><span class="p">.</span><span class="nx">parseCSV</span><span class="o">=</span> <span class="kd">function</span><span class="p">(</span><span class="nx">s</span><span class="p">,</span> <span class="nx">trm</span><span class="p">)</span> <span class="p">{</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-3'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-3'>#</a>
</div>
<p>Get rid of any trailing \n</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">s</span> <span class="o">=</span> <span class="nx">chomp</span><span class="p">(</span><span class="nx">s</span><span class="p">);</span>
<span class="kd">var</span> <span class="nx">cur</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="c1">// The character we are currently processing.</span>
<span class="nx">inQuote</span> <span class="o">=</span> <span class="kc">false</span><span class="p">,</span>
<span class="nx">fieldQuoted</span> <span class="o">=</span> <span class="kc">false</span><span class="p">,</span>
<span class="nx">field</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="c1">// Buffer for building up the current field</span>
<span class="nx">row</span> <span class="o">=</span> <span class="p">[],</span>
<span class="nx">out</span> <span class="o">=</span> <span class="p">[],</span>
<span class="nx">i</span><span class="p">,</span>
<span class="nx">processField</span><span class="p">;</span>
<span class="nx">processField</span> <span class="o">=</span> <span class="kd">function</span> <span class="p">(</span><span class="nx">field</span><span class="p">)</span> <span class="p">{</span>
<span class="k">if</span> <span class="p">(</span><span class="nx">fieldQuoted</span> <span class="o">!==</span> <span class="kc">true</span><span class="p">)</span> <span class="p">{</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-4'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-4'>#</a>
</div>
<p>If field is empty set to null</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="k">if</span> <span class="p">(</span><span class="nx">field</span> <span class="o">===</span> <span class="s1">&#39;&#39;</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">field</span> <span class="o">=</span> <span class="kc">null</span><span class="p">;</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-5'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-5'>#</a>
</div>
<p>If the field was not quoted and we are trimming fields, trim it</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="p">}</span> <span class="k">else</span> <span class="k">if</span> <span class="p">(</span><span class="nx">trm</span> <span class="o">===</span> <span class="kc">true</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">field</span> <span class="o">=</span> <span class="nx">trim</span><span class="p">(</span><span class="nx">field</span><span class="p">);</span>
<span class="p">}</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-6'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-6'>#</a>
</div>
<p>Convert unquoted numbers to their appropriate types</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="k">if</span> <span class="p">(</span><span class="nx">rxIsInt</span><span class="p">.</span><span class="nx">test</span><span class="p">(</span><span class="nx">field</span><span class="p">))</span> <span class="p">{</span>
<span class="nx">field</span> <span class="o">=</span> <span class="nb">parseInt</span><span class="p">(</span><span class="nx">field</span><span class="p">,</span> <span class="mi">10</span><span class="p">);</span>
<span class="p">}</span> <span class="k">else</span> <span class="k">if</span> <span class="p">(</span><span class="nx">rxIsFloat</span><span class="p">.</span><span class="nx">test</span><span class="p">(</span><span class="nx">field</span><span class="p">))</span> <span class="p">{</span>
<span class="nx">field</span> <span class="o">=</span> <span class="nb">parseFloat</span><span class="p">(</span><span class="nx">field</span><span class="p">,</span> <span class="mi">10</span><span class="p">);</span>
<span class="p">}</span>
<span class="p">}</span>
<span class="k">return</span> <span class="nx">field</span><span class="p">;</span>
<span class="p">};</span>
<span class="k">for</span> <span class="p">(</span><span class="nx">i</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span> <span class="nx">i</span> <span class="o">&lt;</span> <span class="nx">s</span><span class="p">.</span><span class="nx">length</span><span class="p">;</span> <span class="nx">i</span> <span class="o">+=</span> <span class="mi">1</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">cur</span> <span class="o">=</span> <span class="nx">s</span><span class="p">.</span><span class="nx">charAt</span><span class="p">(</span><span class="nx">i</span><span class="p">);</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-7'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-7'>#</a>
</div>
<p>If we are at a EOF or EOR</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="k">if</span> <span class="p">(</span><span class="nx">inQuote</span> <span class="o">===</span> <span class="kc">false</span> <span class="o">&amp;&amp;</span> <span class="p">(</span><span class="nx">cur</span> <span class="o">===</span> <span class="s1">&#39;,&#39;</span> <span class="o">||</span> <span class="nx">cur</span> <span class="o">===</span> <span class="s2">&quot;\n&quot;</span><span class="p">))</span> <span class="p">{</span>
<span class="nx">field</span> <span class="o">=</span> <span class="nx">processField</span><span class="p">(</span><span class="nx">field</span><span class="p">);</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-8'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-8'>#</a>
</div>
<p>Add the current field to the current row</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">row</span><span class="p">.</span><span class="nx">push</span><span class="p">(</span><span class="nx">field</span><span class="p">);</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-9'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-9'>#</a>
</div>
<p>If this is EOR append row to output and flush row</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="k">if</span> <span class="p">(</span><span class="nx">cur</span> <span class="o">===</span> <span class="s2">&quot;\n&quot;</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">out</span><span class="p">.</span><span class="nx">push</span><span class="p">(</span><span class="nx">row</span><span class="p">);</span>
<span class="nx">row</span> <span class="o">=</span> <span class="p">[];</span>
<span class="p">}</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-10'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-10'>#</a>
</div>
<p>Flush the field buffer</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">field</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="p">;</span>
<span class="nx">fieldQuoted</span> <span class="o">=</span> <span class="kc">false</span><span class="p">;</span>
<span class="p">}</span> <span class="k">else</span> <span class="p">{</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-11'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-11'>#</a>
</div>
<p>If it's not a ", add it to the field buffer</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="k">if</span> <span class="p">(</span><span class="nx">cur</span> <span class="o">!==</span> <span class="s1">&#39;&quot;&#39;</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">field</span> <span class="o">+=</span> <span class="nx">cur</span><span class="p">;</span>
<span class="p">}</span> <span class="k">else</span> <span class="p">{</span>
<span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nx">inQuote</span><span class="p">)</span> <span class="p">{</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-12'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-12'>#</a>
</div>
<p>We are not in a quote, start a quote</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">inQuote</span> <span class="o">=</span> <span class="kc">true</span><span class="p">;</span>
<span class="nx">fieldQuoted</span> <span class="o">=</span> <span class="kc">true</span><span class="p">;</span>
<span class="p">}</span> <span class="k">else</span> <span class="p">{</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-13'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-13'>#</a>
</div>
<p>Next char is ", this is an escaped "</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="k">if</span> <span class="p">(</span><span class="nx">s</span><span class="p">.</span><span class="nx">charAt</span><span class="p">(</span><span class="nx">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">===</span> <span class="s1">&#39;&quot;&#39;</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">field</span> <span class="o">+=</span> <span class="s1">&#39;&quot;&#39;</span><span class="p">;</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-14'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-14'>#</a>
</div>
<p>Skip the next char</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">i</span> <span class="o">+=</span> <span class="mi">1</span><span class="p">;</span>
<span class="p">}</span> <span class="k">else</span> <span class="p">{</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-15'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-15'>#</a>
</div>
<p>It's not escaping, so end quote</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">inQuote</span> <span class="o">=</span> <span class="kc">false</span><span class="p">;</span>
<span class="p">}</span>
<span class="p">}</span>
<span class="p">}</span>
<span class="p">}</span>
<span class="p">}</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-16'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-16'>#</a>
</div>
<p>Add the last field</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">field</span> <span class="o">=</span> <span class="nx">processField</span><span class="p">(</span><span class="nx">field</span><span class="p">);</span>
<span class="nx">row</span><span class="p">.</span><span class="nx">push</span><span class="p">(</span><span class="nx">field</span><span class="p">);</span>
<span class="nx">out</span><span class="p">.</span><span class="nx">push</span><span class="p">(</span><span class="nx">row</span><span class="p">);</span>
<span class="k">return</span> <span class="nx">out</span><span class="p">;</span>
<span class="p">};</span>
<span class="kd">var</span> <span class="nx">rxIsInt</span> <span class="o">=</span> <span class="sr">/^\d+$/</span><span class="p">,</span>
<span class="nx">rxIsFloat</span> <span class="o">=</span> <span class="sr">/^\d*\.\d+$|^\d+\.\d*$/</span><span class="p">,</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-17'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-17'>#</a>
</div>
<p>If a string has leading or trailing space,
contains a comma double quote or a newline
it needs to be quoted in CSV output</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="nx">rxNeedsQuoting</span> <span class="o">=</span> <span class="sr">/^\s|\s$|,|&quot;|\n/</span><span class="p">,</span>
<span class="nx">trim</span> <span class="o">=</span> <span class="p">(</span><span class="kd">function</span> <span class="p">()</span> <span class="p">{</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-18'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-18'>#</a>
</div>
<p>Fx 3.1 has a native trim function, it's about 10x faster, use it if it exists</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="k">if</span> <span class="p">(</span><span class="nb">String</span><span class="p">.</span><span class="nx">prototype</span><span class="p">.</span><span class="nx">trim</span><span class="p">)</span> <span class="p">{</span>
<span class="k">return</span> <span class="kd">function</span> <span class="p">(</span><span class="nx">s</span><span class="p">)</span> <span class="p">{</span>
<span class="k">return</span> <span class="nx">s</span><span class="p">.</span><span class="nx">trim</span><span class="p">();</span>
<span class="p">};</span>
<span class="p">}</span> <span class="k">else</span> <span class="p">{</span>
<span class="k">return</span> <span class="kd">function</span> <span class="p">(</span><span class="nx">s</span><span class="p">)</span> <span class="p">{</span>
<span class="k">return</span> <span class="nx">s</span><span class="p">.</span><span class="nx">replace</span><span class="p">(</span><span class="sr">/^\s*/</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">).</span><span class="nx">replace</span><span class="p">(</span><span class="sr">/\s*$/</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">);</span>
<span class="p">};</span>
<span class="p">}</span>
<span class="p">}());</span>
<span class="kd">function</span> <span class="nx">chomp</span><span class="p">(</span><span class="nx">s</span><span class="p">)</span> <span class="p">{</span>
<span class="k">if</span> <span class="p">(</span><span class="nx">s</span><span class="p">.</span><span class="nx">charAt</span><span class="p">(</span><span class="nx">s</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">!==</span> <span class="s2">&quot;\n&quot;</span><span class="p">)</span> <span class="p">{</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-19'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-19'>#</a>
</div>
<p>Does not end with \n, just return string</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="k">return</span> <span class="nx">s</span><span class="p">;</span>
<span class="p">}</span> <span class="k">else</span> <span class="p">{</span></pre></div>
</div>
</div>
<div class='clearall'></div>
<div class='section' id='section-20'>
<div class='docs'>
<div class='octowrap'>
<a class='octothorpe' href='#section-20'>#</a>
</div>
<p>Remove the \n</p>
</div>
<div class='code'>
<div class="highlight"><pre> <span class="k">return</span> <span class="nx">s</span><span class="p">.</span><span class="nx">substring</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nx">s</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="mi">1</span><span class="p">);</span>
<span class="p">}</span>
<span class="p">}</span>
<span class="p">}(</span><span class="nx">jQuery</span><span class="p">,</span> <span class="k">this</span><span class="p">.</span><span class="nx">recline</span><span class="p">.</span><span class="nx">Backend</span><span class="p">));</span>
</pre></div>
</div>
</div>
<div class='clearall'></div>
</div>
</body>
Reference in New Issue View Git Blame Copy Permalink