Merge branch '#242/natural-lang-arasaac-api'

ansible/roles/couchDBDockerCouchdb/tasks/main.yml

@@ -41,5 +41,23 @@
   debug:
     var: result
 
+- name: copy files local to remote (vm.args)
+  template:
+    src: vm-args
+    dest: ./couchdb/conf/vm.args
+  register: result
+- name: debug [copy files local to remote with a template (vm.args)]
+  debug:
+    var: result
+
+- name: copy files local to remote (100-couchdb.conf)
+  template:
+    src: 100-couchdb-conf
+    dest: ./couchdb/conf/100-couchdb.conf
+  register: result
+- name: debug [copy files local to remote with a template (100-couchdb.conf)]
+  debug:
+    var: result
+
 - debug:
     msg: END DOCKER COUCHDB...

ansible/roles/couchDBDockerCouchdb/templates/100-couchdb-conf

@@ -0,0 +1,5 @@
+#<domain>    <type>    <item>    <value>
+couchdb      hard      nofile    102400
+couchdb      soft      nofile    102400
+root         hard      nofile    102400
+root         soft      nofile    102400

ansible/roles/couchDBDockerCouchdb/templates/Dockerfile

@@ -1,3 +1,5 @@
 FROM couchdb:3.3.1
 
 COPY ./couchdb/conf/local.ini /opt/couchdb/etc/
+COPY ./couchdb/conf/vm.args /opt/couchdb/etc/
+COPY ./couchdb/conf/100-couchdb.conf /etc/security/limits.d/

ansible/roles/couchDBDockerCouchdb/templates/vm-args

@@ -0,0 +1,38 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+# Ensure that the Erlang VM listens on a known port
+-kernel inet_dist_listen_min 9100
+-kernel inet_dist_listen_max 9100
+
+# Tell kernel and SASL not to log anything
+-kernel error_logger silent
+-sasl sasl_error_logger false
+
+# This will toggle to true in Erlang 25+. However since we don't use global
+# any longer, and have our own auto-connection module, we can keep the
+# existing global behavior to avoid surprises. See
+# https://github.com/erlang/otp/issues/6470#issuecomment-1337421210 for more
+# information about possible increased coordination and messages being sent on
+# disconnections when this setting is enabled.
+#
+-kernel prevent_overlapping_partitions false
+
+# Increase the pool of dirty IO schedulers from 10 to 16
+# Dirty IO schedulers are used for file IO.
++SDio 16
+
+# Comment this line out to enable the interactive Erlang shell on startup
++Bd -noinput
+
+# allow more than 65536 open files, https://docs.couchdb.org/en/main/maintenance                                                                                                                                 /performance.html
++Q 102400

app/lang/i18n.en.json

@@ -919,6 +919,9 @@
   "days": "days",
   "offline": "offline",
   "online": "online",
+  "activateAutomaticGrammarCorrectionARASAACAPI": "Activate automatic grammar correction via ARASAAC's API (available for: {availableLangs})",
+  "noteThatActivatingThisSendsSentencesToARASAACSeePrivacy": "Note that activating this feature results in sending all constructed sentences to ARASAAC's natural language API. See {link}.",
+  "thanksToEduardoLleidaSolanoProfessorAtDepartment": "Thanks to Eduardo Lleida Solano, Professor at the Department of Electronic Engineering and Communications of the University of Zaragoza for collaborating in the development of the ARASAAC API that integrates Natural Language Processing (NLP). It's used to provide a feature for automatically fix the grammar of sentences within AsTeRICS Grid.",
   "couldntPlayWebradioBecauseTheBrowserDidntAllow": "Couldn't play webradio, because the browser didn't allow it.",
   "allowPlayingRadio": "Allow playing radio"
 }

app/privacy_de.html

@@ -105,9 +105,15 @@ <h2>Rechtsgrundlage</h2>
     personenbezogener Datenwert) ist Einwilligung, da bei der Registrierung für einen Online-User ein
     entsprechendes Häkchen zu setzen ist.
 </div>
-<h2>Datenübertragung</h2>
+<h2 id="data-transfer">Datenübertragung</h2>
 <div>
-    Es werden keine Daten an Dritte weitergegeben.
+    Wenn Sie die optionale Funktion der automatischen Grammatikkorrektur durch die ARASAAC API über die Einstellungen aktivieren,
+    werden alle über die App gebildeten Sätze an die API von ARASAAC übertragen. Das geschieht zum Zeitpunkt der Aussprache
+    und zum Zweck der Korrektur der Grammatik.
+    Siehe die <a href="https://arasaac.org/privacy-policy" target="_blank">Datenschutzerklärung von ARASAAC</a> für weitere Details.
+    <br/>
+    <br/>
+    Es werden ansonsten keine Daten an Dritte weitergegeben.
 </div>
 <h2>Speicherdauer</h2>
 <div>
@@ -140,6 +146,10 @@ <h2>Widerruf und Informationen zu Rechten</h2>
     let backHash = url.searchParams.get("back") || "about";
     document.getElementById('backLink1').href = '../#' + backHash;
     document.getElementById('backLink2').href = '../#' + backHash;
+    if (window.location.hash) {
+        console.warn(document.getElementById(window.location.hash.replace('#', '')));
+        document.getElementById(window.location.hash.replace('#', '')).style = 'outline: 3px solid lightblue';
+    }
 </script>
 
 </body>

app/privacy_en.html

@@ -105,9 +105,15 @@ <h2>Legal basis</h2>
     The legal basis for the usage of the username (the only potentially personal data processed by AsTeRICS Grid) is
     consent, since when registering for an "online user" the user has to consent the processing of the data.
 </div>
-<h2>Data transfer</h2>
+<h2 id="data-transfer">Data transfer</h2>
 <div>
-    No data is transferred to third parties.
+    If you're activating the optional function for automatically correct grammar via the API from ARASAAC in the settings,
+    all sentences constructed via the app will be sent to ARASAAC's API. This will happen at the time of speaking out the
+    sentence and the reason for transmitting the data is to provide the function for correcting grammar. For further
+    details see <a href="https://arasaac.org/privacy-policy" target="_blank">ARASAAC's privacy policy</a>.
+    <br/>
+    <br/>
+    Apart from this no data is transferred to third parties.
 </div>
 <h2>Storage period</h2>
 <div>
@@ -140,6 +146,10 @@ <h2>Revocation und information on rights</h2>
     let backHash = url.searchParams.get("back") || "about";
     document.getElementById('backLink1').href = '../#' + backHash;
     document.getElementById('backLink2').href = '../#' + backHash;
+    if (window.location.hash) {
+        console.warn(document.getElementById(window.location.hash.replace('#', '')));
+        document.getElementById(window.location.hash.replace('#', '')).style = 'outline: 3px solid lightblue';
+    }
 </script>
 </body>
 </html>

docs/AG_server_migration.txt

@@ -0,0 +1,42 @@
+# CouchDB-Migration - Plan 1
+1. DONE: Migrate users from sl-users to auth-users in original database
+2. Update new auth server to serve new couch-auth at / not /new/
+3. Use credentials of old database in new auth-server
+4. change DNS for login.couchdb.asterics-foundation.org pointing to new auth server, still using old couchDB server
+5. DOWNTIME: get new certificate for login.couchdb.asterics-foundation.org for new auth server
+6. replicate couchDB data from old server to new couchDB server
+7. change admin password for new couchDB to old couchDB admin password
+8. change DNS for db.couchdb.asterics-foundation.org pointing to new couchdb server
+9. DOWNTIME: get new certificate for db.couchdb.asterics-foundation.org for new couchDB server
+10. shut down old server
+
+# CouchDB-Migration - Plan 2
+1. DONE: Migrate users from sl-users to auth-users in original database
+2. Update new auth server to serve new couch-auth at / not /new/
+3. Use credentials of old database in new auth-server
+4. Release new version of AsTeRICS Grid using login1.asterics-foundation.org for authentication
+5. replicate couchDB data from old server to new couchDB server
+7. change config in auth-server to use db1.asterics-foundation.org instead of db.asterics-foundation.org
+8. Monitor access that is still coming to old server (not updated version of AsTeRICS Grid)
+9. If no access to old server anymore - shut down.
+
+# Open issues
+* validate-username missing in couch-auth -> direkt in start.js von couch-auth mit nano
+* redirect after successful registration (login after registration fails - add timeout?!)
+
+# Questions about Docker / Ansible
+
+## restart: always für superlogin docker-container?! in ansible/roles/superLoginDockerCompose/templates/docker-compose:27
+
+## ansible/roles/couchDBDockerCertbot/tasks/main.yml:53 - reload nginx config only if no certificate - how to update nginx config using ansible?
+
+## ansible tasks "superloginCloneRepoDocker" und "superloginDockerCompose"
+* relationship between "env" file, "Dockerfile" and "docker-compose" file
+* how to update containers, if config changes? (e.g. password of couchDB, hostname, etc.)
+
+## ansible/roles/superLoginDockerNginx/tasks/main.yml:39 docker compose up takes very long
+* docker compose -f ./docker-compose.yml up --build --detach --> what is it doing exactly (I assume npm install?!)
+* possibilities for speed up and logging?!
+
+## run docker commands without sudo?!
+

docs/AG_server_stats.txt

@@ -0,0 +1,84 @@
+node couchDBCompact.js "http://admin:pw@localhost:5984" compact
+
+2.3.1~stretch 500
+
+/dev/mapper/vg00-lv01
+/dev/vg00/lv01
+
+16.11. 09:35 -> /dev/mapper/vg00-lv01   96G   64G   29G  70% /
+16.11. 14:50 -> /dev/mapper/vg00-lv01   96G   60G   33G  65% / nach manual compact
+17.11. 08:54 -> /dev/mapper/vg00-lv01   96G   65G   28G  71% /
+18.11. 09:18 -> /dev/mapper/vg00-lv01   96G   73G   20G  79% /
+18.11. 12:45 -> /dev/mapper/vg00-lv01   96G   61G   32G  66% / nach manual compact
+21.11. 08:36 -> /dev/mapper/vg00-lv01   96G   73G   20G  79% /
+21.11. 14:26 -> /dev/mapper/vg00-lv01   96G   66G   27G  72% / nach manual compact (incomplete)
+22.11. 15:38 -> /dev/mapper/vg00-lv01   96G   78G   15G  85% /
+23.11. 09:30 -> /dev/mapper/vg00-lv01  195G   85G  103G  46% / nach Erhöhung auf 200GB
+23.11. 13:35 -> /dev/mapper/vg00-lv01  195G   70G  118G  38% / nach manual compact
+24.11. 08:40 -> /dev/mapper/vg00-lv01  195G   60G  127G  33% / auto-compact hat funktioniert!
+25.11. 10:02 -> /dev/mapper/vg00-lv01  195G   67G  120G  36% /
+28.11. 08:27 -> /dev/mapper/vg00-lv01  195G   71G  116G  38% /
+29.11. 10:02 -> /dev/mapper/vg00-lv01  195G   78G  110G  42% /
+30.11. 10:01 -> /dev/mapper/vg00-lv01  195G   75G  112G  40% /
+01.12. 16:51 -> /dev/mapper/vg00-lv01  195G   89G   98G  48% /
+02.12. 11:30 -> /dev/mapper/vg00-lv01  195G   84G  103G  45% /
+05.12. 09:46 -> /dev/mapper/vg00-lv01  195G   85G  102G  46% /
+09.12. 15:09 -> /dev/mapper/vg00-lv01  195G   96G   92G  52% /
+13.12. 08:50 -> /dev/mapper/vg00-lv01  195G   96G   92G  52% /
+20.12. 10:15 -> /dev/mapper/vg00-lv01  195G  104G   83G  56% /
+27.12. 08:47 -> /dev/mapper/vg00-lv01  195G  107G   80G  58% /
+02.01. 15:21 -> /dev/mapper/vg00-lv01  195G  112G   75G  61% /
+09.01. 10:32 -> /dev/mapper/vg00-lv01  195G  117G   71G  63% /
+17.01. 09:12 -> /dev/mapper/vg00-lv01  195G  136G   52G  73% /
+17.01. 14:57 -> /dev/mapper/vg00-lv01  195G   89G   98G  48% / nach manual compact
+20.01. 14:46 -> /dev/mapper/vg00-lv01  195G  106G   81G  57% /
+09.02. 08:46 -> /dev/mapper/vg00-lv01  195G  162G   25G  87% /
+09.02. 15:58 -> /dev/mapper/vg00-lv01  195G  115G   72G  62% / nach manual compact
+27.02. 10:23 -> /dev/mapper/vg00-lv01  195G  168G   19G  91% /
+28.02. 17:08 -> /dev/mapper/vg00-lv01  195G  132G   55G  71% / nach manual compact
+10.03. 09:46 -> /dev/mapper/vg00-lv01  195G  186G  928M 100% /
+10.03. 19:29 -> /dev/mapper/vg00-lv01  195G  146G   42G  78% / nach manual compact
+22.03. 10:23 -> /dev/mapper/vg00-lv01  293G  160G  122G  57% / nach resize auf 300GB
+19.04. 10:11 -> /dev/mapper/vg00-lv01  293G  235G   46G  84% /
+19.04. 15:39 -> /dev/mapper/vg00-lv01  293G  178G  103G  64% / nach manual compact
+28.04. 12:53 -> /dev/mapper/vg00-lv01  293G  220G   62G  79% /
+16.05. 14:15 -> /dev/mapper/vg00-lv01  293G  271G   11G  97% /
+17.05. 09:16 -> /dev/mapper/vg00-lv01  293G  208G   74G  74% / nach manual compact
+
+
+
+
+pro User ca. 65GB / 10000 = 6,7MB
+10 000 User in 3 Monaten = ca. 111 neue User pro Tag
+=> ca. 1GB pro Tag
+=> ca. 365GB pro Jahr
+
+17.01. -> 16 333 User
+4,5 Monate = 135 Tage => 120 User pro Tag
+
+
+
+---
+manual compact:
+root@localhost:/var/log/couchdb# cat couchdb.log | grep -c compaction
+89066
+
+auto compact:
+root@localhost:/var/log/couchdb# cat couchdb.log.1 | grep -c compaction
+11307
+
+auto mit snooze_time_ms = 0
+root@localhost:/var/log/couchdb# cat couchdb.log.1 | grep -c compaction
+170737
+
+-----
+30.11.2022
+---------- RESULT ---------
+No metadata: 577
+Single metadata: 11646
+Duplicated metadata: 78
+
+
+
+
+

latest/ansible/roles/couchDBDockerNginx/templates/htpasswd

@@ -0,0 +1 @@
+{{MUNIN_HTPASSWD}}

latest/ansible/roles/superLoginDockerNginx/templates/htpasswd

@@ -0,0 +1 @@
+{{MUNIN_HTPASSWD}}

src/js/model/GridActionCollectElement.js

@@ -18,6 +18,15 @@ class GridActionCollectElement extends Model({
         return 'GridActionCollectElement';
     }
 
+    static isSpeakAction(action) {
+        return [
+            GridActionCollectElement.COLLECT_ACTION_SPEAK,
+            GridActionCollectElement.COLLECT_ACTION_SPEAK_CONTINUOUS,
+            GridActionCollectElement.COLLECT_ACTION_SPEAK_CLEAR,
+            GridActionCollectElement.COLLECT_ACTION_SPEAK_CONTINUOUS_CLEAR
+        ].includes(action);
+    }
+
     static getActions() {
         return [
             GridActionCollectElement.COLLECT_ACTION_SPEAK,

src/js/model/MetaData.js

@@ -22,7 +22,8 @@ class MetaData extends Model({
     colorConfig: [ColorConfig],
     textConfig: [TextConfig],
     localeConfig: [LocaleConfig],
-    notificationConfig: [NotificationConfig]
+    notificationConfig: [NotificationConfig],
+    activateARASAACGrammarAPI: [Boolean]
 }) {
     constructor(properties, elementToCopy) {
         properties = modelUtil.setDefaults(properties, elementToCopy, MetaData) || {};