Merge pull request project-sunbird#3464 from project-sunbird/release-…

…4.10.1

Merge release 4.10.1 to master

ansible/artifacts/sunbird/login/resources/js/telemetry_service.js

@@ -2506,7 +2506,7 @@ if(client_id.toLowerCase() === 'android'){
     "telemetry": {
       "pdata": {
         "id": pdataId,
-        "ver": "4.9.0",
+        "ver": "4.10.3",
         "pid": "sunbird-portal"
       }
     }

ansible/es.yml

@@ -24,11 +24,11 @@
   tags:
     - log_es_backup
 
-- hosts: log-es
-  become: yes
-  vars_files:
-    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-  roles:
-    - es2-data-purge
-  tags:
-    - purge_old_logs
+# - hosts: log-es
+#   become: yes
+#   vars_files:
+#     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+#   roles:
+#     - es2-data-purge
+#   tags:
+#     - purge_old_logs

ansible/graylog-mongodb-import.yml

@@ -1,13 +1,10 @@
 - hosts: graylog
   become: yes
   pre_tasks:
-    - name: Populate service facts
-      ansible.builtin.service_facts:
     - name: stop graylog
       systemd:
         name: graylog-server
         state: stopped
-      when: "'graylog-server' in ansible_facts.services"
 
     - name: wait for graylog to stop completely
       wait_for:
@@ -26,7 +23,6 @@
       systemd:
         name: graylog-server
         state: started
-      when: "'graylog-server' in ansible_facts.services"
 
     - name: Wait for graylog to startup
       uri:
@@ -36,4 +32,3 @@
       until: _result.status == 200
       delay: 15
       retries: 15
-      when: "'graylog-server' in ansible_facts.services"

ansible/graylog.yml

@@ -3,11 +3,11 @@
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
   vars:
-    mongodb_version: "4.0"
+    mongodb_version: "4.4"
     mongodb_replication_replset: graylog
     mongodb_net_bindip: "127.0.0.1,{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}"
     graylog_version: "4.2"
-    graylog_full_version: "4.2.6-1"
+    graylog_full_version: "4.2.9-1"
     mongodb_login_host: "{{ groups['graylog'][0] }}"
     host_name: "{{ groups['graylog'][0] }}"
     graylog_install_elasticsearch: false

ansible/roles/es-mapping/files/indices/orgv2.json → ansible/roles/es-mapping/files/indices/orgv3.json

@@ -47,4 +47,4 @@
   "aliases" : {
     "org_alias" : {}
   }
-}
+}

ansible/roles/es-mapping/files/mappings/orgv2-mapping.json → ansible/roles/es-mapping/files/mappings/orgv3-mapping.json

@@ -158,7 +158,7 @@
           "analyzer": "keylower"
         }
       },
-      "analyzer": "cs_search_analyzer",
+      "analyzer": "cs_index_analyzer",
       "search_analyzer": "cs_search_analyzer"
     },
     "orgType": {

ansible/roles/graylog-mongodb-import/defaults/main.yml

@@ -1,4 +1,4 @@
 graylog_mongo_db: "graylog"
 mongo_master: "{{ groups['graylog'][0] }}"
 mongo_json_data: /tmp/mongo_json_data
-graylog_mongo_collections: "cluster_config,content_packs,content_packs_installations,event_definitions,event_notifications,event_notification_status,grants,grok_patterns,index_sets,inputs,lut_caches,lut_data_adapters,lut_tables,pipeline_processor_pipelines,pipeline_processor_pipelines_streams,pipeline_processor_rules,roles,scheduler_job_definitions,scheduler_triggers,searches,streamrules,streams,users,views"
+graylog_mongo_collections: "cluster_config,content_packs,content_packs_installations,event_definitions,event_notifications,grants,grok_patterns,index_sets,inputs,lut_caches,lut_data_adapters,lut_tables,pipeline_processor_pipelines,pipeline_processor_pipelines_streams,pipeline_processor_rules,roles,scheduler_job_definitions,scheduler_triggers,searches,streamrules,streams,users,views"

ansible/roles/keycloak-deploy/tasks/bootstrap.yml

@@ -28,7 +28,6 @@
   shell: "nohup {{keycloak_home}}/bin/standalone.sh -Dkeycloak.profile.feature.upload_scripts=enabled -b={{ansible_default_ipv4.address}} -bprivate={{ansible_default_ipv4.address}} --server-config standalone-ha.xml &"
   become: yes
   become_user: "{{ wildfly_user }}"
-  run_once: true
   notify: wait for keycloak to start
 
 - meta: flush_handlers
@@ -37,7 +36,6 @@
   apt:
     name: ["python-setuptools"]
     update_cache: true
-  run_once: true
 
 - name: Ensure python packages are installed
   apt: 
@@ -55,38 +53,32 @@
   copy:
     src: "{{ role_path }}/files/python-keycloak-0.12.0"
     dest: /tmp/
-  run_once: true
 
 - name: Initialize python library to run keycloak bootstrap script
   shell: cd /tmp/python-keycloak-0.12.0 && python setup.py install
-  run_once: true
 
 - name: Save keycloak vars to json
   template:
     src: "keycloak-bootstrap.conf.j2"
     dest: "/tmp/keycloak-bootstrap.conf.json"
     mode: "0644"
-  run_once: true
 
 - name: Copy realm json file to tmp location
   template:
     src: "keycloak-realm.j2"
     dest: "/tmp/keycloak-realm.json"
     mode: "0644"
-  run_once: true
 
 - name: Copy user manager roles file to tmp location
   copy:
     src: "files/python-keycloak-0.12.0/roles.json"
     dest: "/tmp/roles.json"
     mode: "0644"
-  run_once: true
 
 - name: Copy the keycloak bootstrap script
   copy:
     src: "{{ role_path }}/files/python-keycloak-0.12.0/keycloak"
     dest: /tmp
-  run_once: true
 
 - name: Run the keycloak bootstrap script
   shell: cd /tmp/keycloak/ && python keycloak_main.py /tmp/keycloak-bootstrap.conf.json

ansible/roles/kong-api/defaults/main.yml

@@ -94,6 +94,7 @@ user_extension_prefix: /user-extension
 uci_admin_prefix: /uci
 notification_service_prefix: /notification
 registry_service_prefix: /rc
+assessment_prefix: /assessment
 
 # Service URLs
 knowledge_mw_service_url: "http://knowledge-mw-service:5000"
@@ -5913,6 +5914,27 @@ kong_apis:
     config.required: false
     config.enabled: false
 
+- name: assessmentItemRead
+  uris: "{{ assessment_prefix }}/v1/items/read"
+  upstream_url: "{{ vm_learning_service_url }}/assessment/v3/items/read"
+  strip_uri: true
+  plugins:
+  - name: jwt
+  - name: cors
+  - "{{ statsd_pulgin }}"
+  - name: acl
+    config.whitelist:
+    - assessmentAccess
+  - name: rate-limiting
+    config.policy: local
+    config.hour: "{{ medium_rate_limit_per_hour }}"
+    config.limit_by: credential
+  - name: request-size-limiting
+    config.allowed_payload_size: "{{ medium_request_size_limit }}"
+  - name: opa-checks
+    config.required: false
+    config.enabled: false
+
 - name: collectionUpdate
   uris: "{{ collection_prefix }}/v1/update"
   upstream_url: "{{ content_service_url }}/collection/v4/update"
@@ -9551,3 +9573,24 @@ kong_apis:
   - name: opa-checks
     config.required: true
     config.enabled: true
+
+- name: getReportDatasets
+  uris: "{{ report_service_prefix }}/report/datasets/get"
+  upstream_url: "{{ report_service_url }}/report/datasets/get"
+  strip_uri: true
+  plugins:
+  - name: jwt
+  - name: cors
+  - "{{ statsd_pulgin }}"
+  - name: acl
+    config.whitelist:
+    - reportsAccess
+  - name: rate-limiting
+    config.policy: local
+    config.hour: "{{ medium_rate_limit_per_hour }}"
+    config.limit_by: credential
+  - name: request-size-limiting
+    config.allowed_payload_size: "{{ small_request_size_limit }}"
+  - name: opa-checks
+    config.required: true
+    config.enabled: true

ansible/roles/ml-analytics-service/defaults/main.yml

@@ -48,6 +48,21 @@ ml_analytics_project_output_dir: "{{ WORKDIR }}/source/projects/output"
 ml_analytics_observation_status_output_dir: "{{ WORKDIR }}/source/observations/status/output"
 ml_analytics_api_authorization_key: "{{ml_api_auth_token | default('sunbird_api_auth_token')}}"
 ml_analytics_api_access_token: "{{ml_api_access_token | default('ml_core_internal_access_token')}}"
-ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_id","school_externalId","state_name","state_id","state_externalId","district_name","district_id","district_externalId","zone_name","zone_id","zone_externalId","parent_name","parent_id","block_name","block_id","block_externalId","cluster_name","cluster_id","cluster_externalId","teacher_name","teacher_id","teacher_externalId","schoolLeader_name","schoolLeader_id","schoolLeader_externalId","taluk_id","taluk_externalId","taluk_name","hub_name","hub_id","hub_externalId","mandal_name","mandal_id","mandal_externalId","region_name","region_id","region_externalId","complex_name","complex_id","complex_externalId","completedDate","beat_name","beat_id","beat_externalId","division_name","division_id","division_externalId","Village/Ward_name","Village/Ward_id","Village/Ward_externalId","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name", "ecm_marked_na","board_name","updatedAt"]},"metricsSpec":[]}}}'
+ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_id","school_externalId","state_name","state_id","state_externalId","district_name","district_id","district_externalId","zone_name","zone_id","zone_externalId","parent_name","parent_id","block_name","block_id","block_externalId","cluster_name","cluster_id","cluster_externalId","teacher_name","teacher_id","teacher_externalId","schoolLeader_name","schoolLeader_id","schoolLeader_externalId","taluk_id","taluk_externalId","taluk_name","hub_name","hub_id","hub_externalId","mandal_name","mandal_id","mandal_externalId","region_name","region_id","region_externalId","complex_name","complex_id","complex_externalId","completedDate","beat_name","beat_id","beat_externalId","division_name","division_id","division_externalId","Village/Ward_name","Village/Ward_id","Village/Ward_externalId","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name", "ecm_marked_na","board_name","updatedAt","organisation_id","user_type"]},"metricsSpec":[]}}}'
 ml_analytics_druid_project_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/projects/sl_projects.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}'
 ml_analytics_azure_sas_token: "{{ sunbird_private_storage_account_key }}"
+ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount/ml_observation_distinctCount_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
+ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain/ml_observation_distinctCount_domain.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
+ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain_criteria/ml_observation_distinctCount_domain_criteria.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
+ml_analytics_druid_distinctCnt_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCount/ml_projects_distinctCount.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
+ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCountPrglevel/ml_projects_distinctCount_prgmlevel.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
+ml_analytics_obs_distinctCnt_azure_blob_path: "observation/distinctCount/"
+ml_analytics_obs_distinctCnt_domain_azure_blob_path: "observation/distinctCount_domain/"
+ml_analytics_obs_distinctCnt_domain_criteria_azure_blob_path: "observation/distinctCount_domain_criteria/"
+ml_analytics_projects_distinctCnt_azure_blob_path: "projects/distinctCount/"
+ml_analytics_obs_distinctCnt_output_dir: "{{ WORKDIR }}/source/observations/distinctCount/output"
+ml_analytics_obs_distinctCnt_domain_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain/output"
+ml_analytics_obs_distinctCnt_domain_criteria_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain_criteria/output"
+ml_analytics_projects_distinctCnt_output_dir: "{{ WORKDIR }}/source/projects/distinctCount/output"
+ml_analytics_projects_distinctCnt_prglevel_output_dir: "{{ WORKDIR }}/source/projects/distinctCountPrglevel/output"
+ml_analytics_projects_distinctCnt_prglevel_azure_blob_path: "projects/distinctCountPrglevel/"

ansible/roles/ml-analytics-service/tasks/main.yml

@@ -40,10 +40,23 @@
     - "{{ BASEPATH }}"
     - "{{ WORKDIR }}"
     - "{{ WORKDIR }}/faust_as_service"
-
-- name: Change user and create working directory under opt dir and install python virtual environment
+
+- name: Delete the virtualenv DIR
+  shell: "rm -rf {{ WORKDIR }}/spark_env"
+  become: true
+
+- name: Install python virtual environment
   shell: "cd {{ WORKDIR }} && virtualenv --python=python3.8 spark_venv"
   become: true
+
+- name: Change the ownership of virtual env
+  become: yes
+  file:
+    path: "{{ WORKDIR }}/spark_env"
+    state: directory
+    owner: "{{ USER }}"
+    group: "{{ USER }}"
+    mode: "0755"
 
 - name: Create necessary logs folders for pipeline
   become: yes
@@ -113,4 +126,4 @@
     user: "{{ USER }}"
     minute: "30"
     hour: "18"
-    job: "{{ BASEPATH }}/ml-analytics-service/run.sh"
+    job: "{{ BASEPATH }}/ml-analytics-service/run.sh > {{ BASEPATH }}/ml-analytics-service/crontab_job.log"