Gitiles
Code Review Sign In
asterix-gerrit.ics.uci.edu / incubator-asterixdb-site / 5f34114603bb17e7c3493d060d88bad69e418cda / . / content / docs / 0.9.0 / ncservice.html
blob: 2779df295cd74ae55277cb0fb469e1a1fe8682b9 [file] [log] [blame]
Ian Maxonbf2c56b2017-01-24 14:14:49 -0800[diff] [blame]1<!DOCTYPE html>
2<!--
Ian Maxond5b11d82017-01-25 10:48:05 -0800[diff] [blame]3 | Generated by Apache Maven Doxia at 2017-01-25
Ian Maxonbf2c56b2017-01-24 14:14:49 -0800[diff] [blame]4 | Rendered using Apache Maven Fluido Skin 1.3.0
5-->
6<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7 <head>
8 <meta charset="UTF-8" />
9 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
Ian Maxond5b11d82017-01-25 10:48:05 -0800[diff] [blame]10 <meta name="Date-Revision-yyyymmdd" content="20170125" />
Ian Maxonbf2c56b2017-01-24 14:14:49 -0800[diff] [blame]11 <meta http-equiv="Content-Language" content="en" />
12 <title>AsterixDB &#x2013; <a id="toc">Table of Contents</a></title>
13 <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" />
14 <link rel="stylesheet" href="./css/site.css" />
15 <link rel="stylesheet" href="./css/print.css" media="print" />
16
17
18 <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script>
19
20
21
Ian Maxonbf2c56b2017-01-24 14:14:49 -0800[diff] [blame]22
Ian Maxonbf2c56b2017-01-24 14:14:49 -0800[diff] [blame]23
24 </head>
25 <body class="topBarDisabled">
26
27
28
29
30 <div class="container-fluid">
31 <div id="banner">
32 <div class="pull-left">
33 <a href="./" id="bannerLeft">
34 <img src="images/asterixlogo.png" alt="AsterixDB"/>
35 </a>
36 </div>
37 <div class="pull-right"> </div>
38 <div class="clear"><hr/></div>
39 </div>
40
41 <div id="breadcrumbs">
42 <ul class="breadcrumb">
43
44
Ian Maxond5b11d82017-01-25 10:48:05 -0800[diff] [blame]45 <li id="publishDate">Last Published: 2017-01-25</li>
Ian Maxonbf2c56b2017-01-24 14:14:49 -0800[diff] [blame]46
47
48
49 <li id="projectVersion" class="pull-right">Version: 0.9.0</li>
50
51 <li class="divider pull-right">|</li>
52
53 <li class="pull-right"> <a href="index.html" title="Documentation Home">
54 Documentation Home</a>
55 </li>
56
57 </ul>
58 </div>
59
60
61 <div class="row-fluid">
62 <div id="leftColumn" class="span3">
63 <div class="well sidebar-nav">
64
65
66 <ul class="nav nav-list">
67 <li class="nav-header">Get Started - Installation</li>
68
69 <li class="active">
70
71 <a href="#"><i class="none"></i>Option 1: using NCService</a>
72 </li>
73
74 <li>
75
76 <a href="install.html" title="Option 2: using Managix">
77 <i class="none"></i>
78 Option 2: using Managix</a>
79 </li>
80
81 <li>
82
83 <a href="yarn.html" title="Option 3: using YARN">
84 <i class="none"></i>
85 Option 3: using YARN</a>
86 </li>
87 <li class="nav-header">AsterixDB Primer</li>
88
89 <li>
90
91 <a href="sqlpp/primer-sqlpp.html" title="Option 1: using SQL++">
92 <i class="none"></i>
93 Option 1: using SQL++</a>
94 </li>
95
96 <li>
97
98 <a href="aql/primer.html" title="Option 2: using AQL">
99 <i class="none"></i>
100 Option 2: using AQL</a>
101 </li>
102 <li class="nav-header">Data Model</li>
103
104 <li>
105
106 <a href="datamodel.html" title="The Asterix Data Model">
107 <i class="none"></i>
108 The Asterix Data Model</a>
109 </li>
110 <li class="nav-header">Queries - SQL++</li>
111
112 <li>
113
114 <a href="sqlpp/manual.html" title="The SQL++ Query Language">
115 <i class="none"></i>
116 The SQL++ Query Language</a>
117 </li>
118
119 <li>
120
121 <a href="sqlpp/builtins.html" title="Builtin Functions">
122 <i class="none"></i>
123 Builtin Functions</a>
124 </li>
125 <li class="nav-header">Queries - AQL</li>
126
127 <li>
128
129 <a href="aql/manual.html" title="The Asterix Query Language (AQL)">
130 <i class="none"></i>
131 The Asterix Query Language (AQL)</a>
132 </li>
133
134 <li>
135
136 <a href="aql/builtins.html" title="Builtin Functions">
137 <i class="none"></i>
138 Builtin Functions</a>
139 </li>
140 <li class="nav-header">Advanced Features</li>
141
142 <li>
143
144 <a href="aql/similarity.html" title="Support of Similarity Queries">
145 <i class="none"></i>
146 Support of Similarity Queries</a>
147 </li>
148
149 <li>
150
151 <a href="aql/fulltext.html" title="Support of Full-text Queries">
152 <i class="none"></i>
153 Support of Full-text Queries</a>
154 </li>
155
156 <li>
157
158 <a href="aql/externaldata.html" title="Accessing External Data">
159 <i class="none"></i>
160 Accessing External Data</a>
161 </li>
162
163 <li>
164
165 <a href="feeds/tutorial.html" title="Support for Data Ingestion">
166 <i class="none"></i>
167 Support for Data Ingestion</a>
168 </li>
169
170 <li>
171
172 <a href="udf.html" title="User Defined Functions">
173 <i class="none"></i>
174 User Defined Functions</a>
175 </li>
176
177 <li>
178
179 <a href="aql/filters.html" title="Filter-Based LSM Index Acceleration">
180 <i class="none"></i>
181 Filter-Based LSM Index Acceleration</a>
182 </li>
183 <li class="nav-header">API/SDK</li>
184
185 <li>
186
187 <a href="api.html" title="HTTP API">
188 <i class="none"></i>
189 HTTP API</a>
190 </li>
191 </ul>
192
193
194
195 <hr class="divider" />
196
197 <div id="poweredBy">
198 <div class="clear"></div>
199 <div class="clear"></div>
200 <div class="clear"></div>
201 <a href="./" title="AsterixDB" class="builtBy">
202 <img class="builtBy" alt="AsterixDB" src="images/asterixlogo.png" />
203 </a>
204 </div>
205 </div>
206 </div>
207
208
209 <div id="bodyColumn" class="span9" >
210
211 <!-- ! Licensed to the Apache Software Foundation (ASF) under one
212 ! or more contributor license agreements. See the NOTICE file
213 ! distributed with this work for additional information
214 ! regarding copyright ownership. The ASF licenses this file
215 ! to you under the Apache License, Version 2.0 (the
216 ! "License"); you may not use this file except in compliance
217 ! with the License. You may obtain a copy of the License at
218 !
219 ! http://www.apache.org/licenses/LICENSE-2.0
220 !
221 ! Unless required by applicable law or agreed to in writing,
222 ! software distributed under the License is distributed on an
223 ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
224 ! KIND, either express or implied. See the License for the
225 ! specific language governing permissions and limitations
226 ! under the License.
227 ! --><div class="section">
228<h2><a name="Table_of_Contents"></a><a name="toc" id="toc">Table of Contents</a></h2>
229
230<ul>
231
232<li><a href="#Small_cluster">Starting a small cluster using the NCService</a></li>
233
234<li><a href="#Parameters">Parameter setting</a></li>
235</ul>
236<h1><a name="Small_cluster" id="Small_cluster">Starting a small cluster using the NCService</a></h1>
237<p>When running a cluster using the <tt>NCService</tt> there are 3 different kind of processes involved:</p>
238
239<ol style="list-style-type: decimal">
240
241<li><tt>NCDriver</tt> does the work of a NodeController</li>
242
243<li><tt>NCService</tt> configures and starts an <tt>NCDriver</tt></li>
244
245<li><tt>CCDriver</tt> does the work of a ClusterController and sends the configuration to the <tt>NCServices</tt></li>
246</ol>
247<p>To start a small cluster consisting of 2 NodeControllers (<tt>red</tt> and <tt>blue</tt>) and 1 ClusterController (<tt>cc</tt>) on a single machine only 2 configuration files are required. The first one is</p>
248<p><tt>blue.conf</tt>:</p>
249
250<div class="source">
251<div class="source">
252<pre>[ncservice]
253port=9091
254</pre></div></div>
255<p>It is a configuration file for the second <tt>NCService</tt>. This contains only the port that the <tt>NCService</tt> of the second NodeControllers listens to as it is non-standard. The first <tt>NCService</tt> does not need a configuration file, as it only uses default parameters. In a distributed environment with 1 NodeController per machine, no <tt>NCService</tt> needs a configuration file.</p>
256<p>The second configuration file is</p>
257<p><tt>cc.conf</tt>:</p>
258
259<div class="source">
260<div class="source">
261<pre>[nc/red]
262txnlogdir=/tmp/asterix/red/txnlog
263coredumpdir=/tmp/asterix/red/coredump
264iodevices=/tmp/asterix/red
265
266[nc/blue]
267port=9091
268txnlogdir=/tmp/asterix/blue/txnlog
269coredumpdir=/tmp/asterix/blue/coredump
270iodevices=/tmp/asterix/blue
271
272[nc]
273app.class=org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint
274storagedir=storage
275address=127.0.0.1
276command=asterixnc
277
278[cc]
279cluster.address = 127.0.0.1
280http.port = 12345
281</pre></div></div>
282<p>This is the configuration file for the cluster and it contains information that each <tt>NCService</tt> will use when starting the corresponding <tt>NCDriver</tt> as well as information for the <tt>CCDriver</tt>.</p>
283<p>To start the cluster simply use the following steps</p>
284
285<ol style="list-style-type: decimal">
286
287<li>
288<p>Set BASEDIR to location of an unzipped asterix-server binary assembly (in the source tree that&#x2019;s at <tt>asterixdb/asterix-server/target</tt>).</p>
289
290<div class="source">
291<div class="source">
292<pre>$ export BASEDIR=[..]/asterix-server-0.8.9-SNAPSHOT-binary-assembly
293</pre></div></div></li>
294
295<li>
296<p>Start the 2 <tt>NCServices</tt> for <tt>red</tt> and <tt>blue</tt>.</p>
297
298<div class="source">
299<div class="source">
300<pre>$ $BASEDIR/bin/asterixncservice -config-file blue.conf &gt; blue-service.log 2&gt;&amp;1 &amp;
301$ $BASEDIR/bin/asterixncservice &gt;red-service.log 2&gt;&amp;1 &amp;
302</pre></div></div></li>
303
304<li>
305<p>Start the <tt>CCDriver</tt>.</p>
306
307<div class="source">
308<div class="source">
309<pre>$ $BASEDIR/bin/asterixcc -config-file cc.conf &gt; cc.log 2&gt;&amp;1 &amp;
310</pre></div></div></li>
311</ol>
312<p>The <tt>CCDriver</tt> will connect to the <tt>NCServices</tt> and thus initiate the configuration and the start of the <tt>NCDrivers</tt>. After running these scripts, <tt>jps</tt> should show a result similar to this:</p>
313
314<div class="source">
315<div class="source">
316<pre>$ jps
31713184 NCService
31813200 NCDriver
31913185 NCService
32013186 CCDriver
32113533 Jps
32213198 NCDriver
323</pre></div></div>
324<p>The logs for the <tt>NCDrivers</tt> will be in <tt>$BASEDIR/logs</tt>.</p>
325<p>To stop the cluster again simply run</p>
326
327<div class="source">
328<div class="source">
329<pre>$ kill `jps | egrep '(CDriver|NCService)' | awk '{print $1}'`
330</pre></div></div>
331<p>to kill all processes.</p>
332<h1><a name="Parameters" id="Parameters">Parameter settings</a></h1>
333<p>The following parameters are for the master process, under the &#x201c;[cc]&#x201d; section.</p>
334
335<table border="0" class="table table-striped">
336 <thead>
337
338<tr class="a">
339
340<th>Parameter </th>
341
342<th>Meaning </th>
343
344<th>Default </th>
345 </tr>
346 </thead>
347 <tbody>
348
349<tr class="b">
350
351<td>instance.name </td>
352
353<td>The name of the AsterixDB instance </td>
354
355<td>&#x201c;DEFAULT_INSTANCE&#x201d; </td>
356 </tr>
357
358<tr class="a">
359
360<td>max.wait.active.cluster </td>
361
362<td>The max pending time (in seconds) for cluster startup. After the threshold, if the cluster still is not up and running, it is considered unavailable. </td>
363
364<td>60 </td>
365 </tr>
366
367<tr class="b">
368
369<td>metadata.callback.port </td>
370
371<td>The port for metadata communication </td>
372
373<td>0 </td>
374 </tr>
375
376<tr class="a">
377
378<td>cluster.address </td>
379
380<td>The binding IP address for the AsterixDB instance </td>
381
382<td>N/A </td>
383 </tr>
384 </tbody>
385</table>
386<p>The following parameters for slave processes, under &#x201c;[nc]&#x201d; sections.</p>
387
388<table border="0" class="table table-striped">
389 <thead>
390
391<tr class="a">
392
393<th>Parameter </th>
394
395<th>Meaning </th>
396
397<th>Default </th>
398 </tr>
399 </thead>
400 <tbody>
401
402<tr class="b">
403
404<td>address </td>
405
406<td>The binding IP address for the slave process </td>
407
408<td>N/A </td>
409 </tr>
410
411<tr class="a">
412
413<td>command </td>
414
415<td>The command for the slave process </td>
416
417<td>N/A (for AsterixDB, it should be &#x201c;asterixnc&#x201d;) </td>
418 </tr>
419
420<tr class="b">
421
422<td>coredumpdir </td>
423
424<td>The path for core dump </td>
425
426<td>N/A </td>
427 </tr>
428
429<tr class="a">
430
431<td>iodevices </td>
432
433<td>Comma separated directory paths for both storage files and temporary files </td>
434
435<td>N/A </td>
436 </tr>
437
438<tr class="b">
439
440<td>jvm.args </td>
441
442<td>The JVM arguments </td>
443
444<td>-Xmx1536m </td>
445 </tr>
446
447<tr class="a">
448
449<td>metadata.port </td>
450
451<td>The metadata communication port on the metadata node. This parameter should only be present in the section of the metadata NC </td>
452
453<td>0 </td>
454 </tr>
455
456<tr class="b">
457
458<td>metadata.registration.timeout.secs </td>
459
460<td>The time out threshold (in seconds) for metadata node registration </td>
461
462<td>60 </td>
463 </tr>
464
465<tr class="a">
466
467<td>port </td>
468
469<td>The port for the NCService that starts the slave process </td>
470
471<td>N/A </td>
472 </tr>
473
474<tr class="b">
475
476<td>storagedir </td>
477
478<td>The directory for storage files </td>
479
480<td>N/A </td>
481 </tr>
482
483<tr class="a">
484
485<td>storage.buffercache.maxopenfiles </td>
486
487<td>The maximum number of open files for the buffer cache. Note that this is the parameter for the AsterixDB and setting the operating system parameter is still required. </td>
488
489<td>2147483647 </td>
490 </tr>
491
492<tr class="b">
493
494<td>storage.buffercache.pagesize </td>
495
496<td>The page size (in bytes) for the disk buffer cache (for reads) </td>
497
498<td>131072 </td>
499 </tr>
500
501<tr class="a">
502
503<td>storage.buffercache.size </td>
504
505<td>The overall budget (in bytes) of the disk buffer cache (for reads) </td>
506
507<td>536870912 </td>
508 </tr>
509
510<tr class="b">
511
512<td>storage.lsm.bloomfilter.falsepositiverate </td>
513
514<td>The false positive rate for the bloom filter for each memory/disk components </td>
515
516<td>0.01 </td>
517 </tr>
518
519<tr class="a">
520
521<td>storage.memorycomponent.globalbudget </td>
522
523<td>The global budget (in bytes) for all memory components of all datasets and indexes (for writes) </td>
524
525<td>536870912 </td>
526 </tr>
527
528<tr class="b">
529
530<td>storage.memorycomponent.numcomponents </td>
531
532<td>The number of memory components per data partition per index </td>
533
534<td>2 </td>
535 </tr>
536
537<tr class="a">
538
539<td>storage.memorycomponent.numpages </td>
540
541<td>The number of pages for all memory components of a dataset, including those for secondary indexes </td>
542
543<td>256 </td>
544 </tr>
545
546<tr class="b">
547
548<td>storage.memorycomponent.pagesize </td>
549
550<td>The page size (in bytes) of memory components </td>
551
552<td>131072 </td>
553 </tr>
554
555<tr class="a">
556
557<td>storage.metadata.memorycomponent.numpages </td>
558
559<td>The number of pages for all memory components of a metadata dataset </td>
560
561<td>256 </td>
562 </tr>
563
564<tr class="b">
565
566<td>txnlogdir </td>
567
568<td>The directory for transaction logs </td>
569
570<td>N/A </td>
571 </tr>
572
573<tr class="a">
574
575<td>txn.commitprofiler.reportinterval </td>
576
577<td>The interval for reporting commit statistics </td>
578
579<td>5 </td>
580 </tr>
581
582<tr class="b">
583
584<td>txn.job.recovery.memorysize </td>
585
586<td>The memory budget (in bytes) used for recovery </td>
587
588<td>67108864 </td>
589 </tr>
590
591<tr class="a">
592
593<td>txn.lock.timeout.sweepthreshold </td>
594
595<td>Interval (in milliseconds) for checking lock timeout </td>
596
597<td>10000 </td>
598 </tr>
599
600<tr class="b">
601
602<td>txn.lock.timeout.waitthreshold </td>
603
604<td>Time out (in milliseconds) of waiting for a lock </td>
605
606<td>60000 </td>
607 </tr>
608
609<tr class="a">
610
611<td>txn.log.buffer.numpages </td>
612
613<td>The number of pages in the transaction log tail </td>
614
615<td>8 </td>
616 </tr>
617
618<tr class="b">
619
620<td>txn.log.buffer.pagesize </td>
621
622<td>The page size (in bytes) for transaction log buffer. </td>
623
624<td>131072 </td>
625 </tr>
626
627<tr class="a">
628
629<td>txn.log.checkpoint.history </td>
630
631<td>The number of checkpoints to keep in the transaction log </td>
632
633<td>0 </td>
634 </tr>
635
636<tr class="b">
637
638<td>txn.log.checkpoint.lsnthreshold </td>
639
640<td>The checkpoint threshold (in terms of LSNs (log sequence numbers) that have been written to the transaction log, i.e., the length of the transaction log) for transection logs </td>
641
642<td>67108864 </td>
643 </tr>
644 </tbody>
645</table>
646<p>The following parameter is for both master and slave processes, under the &#x201c;[app]&#x201d; section.</p>
647
648<table border="0" class="table table-striped">
649 <thead>
650
651<tr class="a">
652
653<th>Parameter </th>
654
655<th>Meaning </th>
656
657<th>Default </th>
658 </tr>
659 </thead>
660 <tbody>
661
662<tr class="b">
663
664<td>log.level </td>
665
666<td>The logging level for master and slave processes </td>
667
668<td>&#x201c;INFO&#x201d; </td>
669 </tr>
670
671<tr class="a">
672
673<td>compiler.framesize </td>
674
675<td>The page size (in bytes) for computation </td>
676
677<td>32768 </td>
678 </tr>
679
680<tr class="b">
681
682<td>compiler.groupmemory </td>
683
684<td>The memory budget (in bytes) for a group by operator instance in a partition </td>
685
686<td>33554432 </td>
687 </tr>
688
689<tr class="a">
690
691<td>compiler.joinmemory </td>
692
693<td>The memory budget (in bytes) for a join operator instance in a partition </td>
694
695<td>33554432 </td>
696 </tr>
697
698<tr class="b">
699
700<td>compiler.sortmemory </td>
701
702<td>The memory budget (in bytes) for a sort operator instance in a partition </td>
703
704<td>33554432 </td>
705 </tr>
706
707<tr class="a">
708
709<td>compiler.parallelism </td>
710
711<td>The degree of parallelism for query execution. Zero means to use the storage parallelism as the query execution parallelism, while other integer values dictate the number of query execution parallel partitions. The system will fall back to use the number of all available CPU cores in the cluster as the degree of parallelism if the number set by a user is too large or too small. </td>
712
713<td>0 </td>
714 </tr>
715 </tbody>
716</table></div>
717 </div>
718 </div>
719 </div>
720
721 <hr/>
722
723 <footer>
724 <div class="container-fluid">
725 <div class="row span12">Copyright &copy; 2017
726 <a href="https://www.apache.org/">The Apache Software Foundation</a>.
727 All Rights Reserved.
728
729 </div>
730
731 <?xml version="1.0" encoding="UTF-8"?>
732<div class="row-fluid">Apache AsterixDB, AsterixDB, Apache, the Apache
733 feather logo, and the Apache AsterixDB project logo are either
734 registered trademarks or trademarks of The Apache Software
735 Foundation in the United States and other countries.
736 All other marks mentioned may be trademarks or registered
737 trademarks of their respective owners.</div>
738
739
740 </div>
741 </footer>
742 </body>
743</html>