diff --git a/experimental/aliases.json b/experimental/aliases.json new file mode 100644 index 0000000..fea2c29 --- /dev/null +++ b/experimental/aliases.json @@ -0,0 +1,872 @@ +{ + "askbot": { + "raw": [ + { + "alias": "askbot-raw" + } + ], + "enrich": [ + { + "alias": "askbot" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "apache": { + "raw": [ + { + "alias": "apache" + } + ] + }, + "bugzilla": { + "raw": [ + { + "alias": "bugzilla-raw" + } + ], + "enrich": [ + { + "alias": "bugzilla" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + }, + { + "alias": "all_enriched_tickets" + } + ] + }, + "bugzillarest": { + "raw": [ + { + "alias": "bugzilla-raw" + } + ], + "enrich": [ + { + "alias": "bugzilla" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + }, + { + "alias": "all_enriched_tickets" + } + ] + }, + "crates": { + "raw": [ + { + "alias": "crates-raw" + } + ], + "enrich": [ + { + "alias": "crate" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "confluence": { + "raw": [ + { + "alias": "confluence-raw" + } + ], + "enrich": [ + { + "alias": "confluence" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "discourse": { + "raw": [ + { + "alias": "discourse-raw" + } + ], + "enrich": [ + { + "alias": "discourse" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "dockerhub": { + "raw": [ + { + "alias": "dockerhub-raw" + } + ], + "enrich": [ + { + "alias": "dockerhub" + } + ] + }, + "finosmeetings": { + "raw": [ + { + "alias": "finosmeetings-raw" + } + ], + "enrich": [ + { + "alias": "finosmeetings" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "functest": { + "raw": [ + { + "alias": "functest-raw" + } + ], + "enrich": [ + { + "alias": "functest" + } + ] + }, + "gerrit": { + "raw": [ + { + "alias": "gerrit-raw" + } + ], + "enrich": [ + { + "alias": "gerrit" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "git": { + "raw": [ + { + "alias": "git-raw" + } + ], + "enrich": [ + { + "alias": "git" + }, + { + "alias": "git_author" + }, + { + "alias": "git_enrich" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "githubql": { + "raw": [ + { + "alias": "github_events-raw" + } + ], + "enrich": [ + { + "alias": "github_events" + } + ] + }, + "github:repo": { + "raw": [ + { + "alias": "github_repositories-raw" + } + ], + "enrich": [ + { + "alias": "github_repositories" + } + ] + }, + "github2:issue": { + "raw": [ + { + "alias": "github2_issues-raw" + } + ], + "enrich": [ + { + "alias": "github2_issues" + }, + { + "alias": "github2_pull_requests", + "filter": { + "terms": { + "issue_pull_request" : [ + "true" + ] + } + } + } + ] + }, + "github2:pull": { + "raw": [ + { + "alias": "github2_pull_requests-raw" + } + ], + "enrich": [ + { + "alias": "github2_pull_requests" + } + ] + }, + "github": { + "raw": [ + { + "alias": "github-raw" + } + ], + "enrich": [ + { + "alias": "github_issues" + }, + { + "alias": "github_issues_enrich" + }, + { + "alias": "issues_closed" + }, + { + "alias": "issues_created" + }, + { + "alias": "issues_updated" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + }, + { + "alias": "all_enriched_tickets", + "filter" : { + "terms" : { + "pull_request" : ["false"] + } + } + }, + { + "alias": "github_issues_onion-src", + "filter" : { + "terms" : { + "pull_request" : [ + "false" + ] + } + } + }, + { + "alias": "github_prs_onion-src", + "filter" : { + "terms" : { + "pull_request" : [ + "true" + ] + } + } + } + ] + }, + "gitlab:issue": { + "raw": [ + { + "alias": "gitlab_issues-raw" + } + ], + "enrich": [ + { + "alias": "gitlab_issues" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + }, + { + "alias": "all_enriched_tickets" + } + ] + }, + "gitlab:merge": { + "raw": [ + { + "alias": "gitlab_merge_requests-raw" + } + ], + "enrich": [ + { + "alias": "gitlab_merge_requests" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "gitter": { + "raw": [ + { + "alias": "gitter-raw" + } + ], + "enrich": [ + { + "alias": "gitter" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "google_hits": { + "raw": [ + { + "alias": "google-hits-raw" + } + ], + "enrich": [ + { + "alias": "google-hits" + } + ] + }, + "hyperkitty": { + "raw": [ + { + "alias": "hyperkitty-raw" + } + ], + "enrich": [ + { + "alias": "mbox" + }, + { + "alias": "kafka" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "groupsio": { + "raw": [ + { + "alias": "groupsio-raw" + } + ], + "enrich": [ + { + "alias": "mbox" + }, + { + "alias": "kafka" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "jenkins": { + "raw": [ + { + "alias": "jenkins-raw" + } + ], + "enrich": [ + { + "alias": "jenkins" + } + ] + }, + "jira": { + "raw": [ + { + "alias": "jira-raw" + } + ], + "enrich": [ + { + "alias": "jira" + }, + { + "alias": "jira_resolution_date" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + }, + { + "alias": "all_enriched_tickets" + } + ] + }, + "kitsune": { + "raw": [ + { + "alias": "kitsune-raw" + } + ], + "enrich": [ + { + "alias": "kitsune" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "mattermost": { + "raw": [ + { + "alias": "mattermost-raw" + } + ], + "enrich": [ + { + "alias": "mattermost" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "mbox": { + "raw": [ + { + "alias": "mbox-raw" + } + ], + "enrich": [ + { + "alias": "mbox" + }, + { + "alias": "kafka" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "mediawiki": { + "raw": [ + { + "alias": "mediawiki-raw" + } + ], + "enrich": [ + { + "alias": "mediawiki" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "meetup": { + "raw": [ + { + "alias": "meetup-raw" + } + ], + "enrich": [ + { + "alias": "meetup" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "mozillaclub": { + "raw": [ + { + "alias": "mozillaclub-raw" + } + ], + "enrich": [ + { + "alias": "mozillaclub" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "nntp": { + "raw": [ + { + "alias": "mozillaclub-raw" + } + ], + "enrich": [ + { + "alias": "mbox" + }, + { + "alias": "kafka" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "pagure": { + "raw": [ + { + "alias": "pagure-raw" + } + ], + "enrich": [ + { + "alias": "pagure" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "phabricator": { + "raw": [ + { + "alias": "phabricator-raw" + } + ], + "enrich": [ + { + "alias": "phabricator" + }, + { + "alias": "maniphest" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "pipermail": { + "raw": [ + { + "alias": "pipermail-raw" + } + ], + "enrich": [ + { + "alias": "mbox" + }, + { + "alias": "kafka" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "puppetforge": { + "raw": [ + { + "alias": "puppetforge-raw" + } + ], + "enrich": [ + { + "alias": "puppetforge" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "redmine": { + "raw": [ + { + "alias": "redmine-raw" + } + ], + "enrich": [ + { + "alias": "redmine" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + }, + { + "alias": "all_enriched_tickets" + } + ] + }, + "remo": { + "raw": [ + { + "alias": "remo-raw" + } + ], + "enrich": [ + { + "alias": "remo" + }, + { + "alias": "remo-events" + }, + { + "alias": "remo-events_metadata__timestamp" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "remo:activities": { + "raw": [ + { + "alias": "remo_activities-raw" + } + ], + "enrich": [ + { + "alias": "remo-activities" + }, + { + "alias": "remo-activities_metadata__timestamp" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "rss": { + "raw": [ + { + "alias": "rss-raw" + } + ], + "enrich": [ + { + "alias": "rss" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "slack": { + "raw": [ + { + "alias": "slack-raw" + } + ], + "enrich": [ + { + "alias": "slack" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "stackexchange": { + "raw": [ + { + "alias": "stackexchange-raw" + } + ], + "enrich": [ + { + "alias": "stackoverflow" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "supybot": { + "raw": [ + { + "alias": "irc-raw" + } + ], + "enrich": [ + { + "alias": "irc" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "telegram": { + "raw": [ + { + "alias": "telegram-raw" + } + ], + "enrich": [ + { + "alias": "telegram" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "twitter": { + "raw": [ + { + "alias": "twitter-raw" + } + ], + "enrich": [ + { + "alias": "twitter" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + }, + "weblate": { + "raw": [ + { + "alias": "weblate-raw" + } + ], + "enrich": [ + { + "alias": "weblate" + }, + { + "alias": "affiliations" + }, + { + "alias": "all_enriched" + } + ] + } +} diff --git a/experimental/credentials-example.cfg b/experimental/credentials-example.cfg new file mode 100644 index 0000000..b7397b8 --- /dev/null +++ b/experimental/credentials-example.cfg @@ -0,0 +1,35 @@ +[es_collection] +url = https://es_user:es_password@es_host:9200 + +[es_enrichment] +url = https://es_user:es_password@es_host:9200 + +[sortinghat] +host = +user = +password = +database = + +[github] +api-token = + +[github:repo] +api-token = + +[github:pull] +api-token = + +[github2:issue] +api-token = + +[github2:pull] +api-token = + +[discourse] +# Need a user name even with the global token. +api-username = +api-token = + +[slack] +# Legacy token, works again after upgrading mordred and running docker commit to make a new image +api-token = diff --git a/experimental/docker-compose.yml b/experimental/docker-compose.yml new file mode 100644 index 0000000..fae2b4e --- /dev/null +++ b/experimental/docker-compose.yml @@ -0,0 +1,77 @@ +mordred: + # This version was recommended in CHAOSS slack as it works with + # AWS's open source version of Elasticsearch + image: bitergia/mordred:grimoirelab-0.2.57 + volumes: + - ./setup.cfg:/home/bitergia/conf/setup.cfg + - ./credentials.cfg:/home/bitergia/conf/credentials.cfg + - ./simple-projects.json:/home/bitergia/conf/simple-projects.json + - ./aliases.json:/home/bitergia/conf/aliases.json + environment: + - MORDRED_CONF_FILES=conf/setup.cfg conf/credentials.cfg + links: + - mariadb + - elasticsearch + - kibiter + +hatstall: + image: grimoirelab/hatstall:latest + environment: + - DATABASE_DIR=/db/ + - ADMIN_USER=${HATSTALL_USER} + - ADMIN_PASS=${HATSTALL_PASSWORD} + ports: + - 8000:80 + links: + - mariadb + volumes: + - ./grimoire-config/apache-hatstall.conf:/home/grimoirelab/apache-hatstall.conf + - ./grimoire-config/shdb.cfg:/home/grimoirelab/shdb.cfg + +mariadb: + restart: "always" + image: mariadb:10.0 + expose: + - "3306" + ports: + - "3306:3306" + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_ALLOW_EMPTY_PASSWORD=no + +elasticsearch: + restart: on-failure:5 + image: bitergia/elasticsearch:6.8.6-secured + command: elasticsearch -Enetwork.bind_host=0.0.0.0 -Ehttp.max_content_length=2000mb + environment: + - ES_JAVA_OPTS=-Xms2g -Xmx2g + - ELASTIC_USERNAME=${ELASTIC_USERNAME} + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - xpack.security.enabled=${ELASTIC_SECURITY} + # Results in SSL issue + # WARNING - Something went wrong when adding an alias on + # https://elasticsearch:9200/github_repositories-enriched. + # Alias not set. 400 Client Error: Bad Request + # Probably can be ignored, want to be sure. + #- xpack.security.transport.ssl.enabled=true + expose: + - "9200" + ports: + - "9200:9200" + +kibiter: + restart: on-failure:5 + image: bitergia/kibiter:secured-v6.8.6-3 + environment: + - ANONYMOUS_USER=true + - PROJECT_NAME=Development + - NODE_OPTIONS=--max-old-space-size=1000 + - ELASTICSEARCH_URL=https://elasticsearch:9200 + - ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME} + - ELASTICSEARCH_USER=${ELASTIC_USERNAME} + - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD} + links: + - elasticsearch + ports: + - "5601:5601" + diff --git a/experimental/dot_env b/experimental/dot_env new file mode 100644 index 0000000..4a2af34 --- /dev/null +++ b/experimental/dot_env @@ -0,0 +1,13 @@ +# Update this file with the appropriate values for your environment +# and rename to .env so docker-compose can expand the values and +# reference them from docker-compose.yml +ELASTIC_SECURITY=true +# There's some bug that's preventing the password from updating when compose +# starts. It's being passed in the environment correctly, but not +# updated on the ElasticSearch side. +ELASTIC_PASSWORD= +ELASTIC_USERNAME= +MYSQL_ROOT_PASSWORD= + +HATSTALL_USER= +HATSTALL_PASSWORD= diff --git a/experimental/setup.cfg b/experimental/setup.cfg new file mode 100644 index 0000000..66d735e --- /dev/null +++ b/experimental/setup.cfg @@ -0,0 +1,162 @@ +[general] +short_name = Grimoire +update = true +debug = false +logs_dir = /tmp +aliases_file = conf/aliases.json + +[projects] +projects_file = conf/simple-projects.json + +[es_collection] +# Loaded from credentials.cfg +# url = https://es_user:es_password@elasticsearch:9200 + +[es_enrichment] +# Loaded from credentials.cfg +# url = https://es_user:es_password@elasticsearch:9200 + +[sortinghat] +# Loaded from credentials.cfg +# host = +# user = +# password = +# database = +load_orgs = true +orgs_file = conf/orgs.json +identities_file = [conf/identities.json] +matching = [email-name, github] +autoprofile = [customer, git, github] +sleep_for = 86400 +unaffiliated_group = Unknown +affiliate = True + +[panels] +kibiter_url = http://kibiter:5601 +kibiter_time_from = now-1y +kibiter_default_index = "git" +github-repos = true + +[phases] +collection = true +identities = true +enrichment = true +panels = true + +## +## The protocol for defining indexes will be +## [description_of_source_with_underscores]-[raw|enriched] +## For example: +## git-raw or git-enriched +## gitub_prs-raw or github_prs-enriched +## +[git] +raw_index = git-raw +enriched_index = git-enriched +studies = [enrich_demography:git,enrich_git_branches:git,enrich_areas_of_code:git, enrich_onion:git,enrich_forecast_activity] + +[enrich_demography:git] + +[enrich_areas_of_code:git] +in_index = git-raw +out_index = git_aoc-enriched + +[enrich_git_branches:git] +run_month_days = [4, 23] + +[enrich_onion:git] +in_index = git-raw +out_index = git_onion-enriched +contribs_field = hash + +[enrich_forecast_activity] +out_index = git_study_forecast + +# All things github +[github] +# Loaded from credentials.cfg +# api-token = +raw_index = github_raw +enriched_index = github_enriched +sleep-for-rate = true +no-archive = true +sleep-time = 300 +studies = [enrich_onion:github,enrich_geolocation:user,enrich_geolocation:assignee,enrich_backlog_analysis] + +[github:repo] +# Loaded from credentials.cfg +# api-token = +raw_index = github-repo_raw +enriched_index = github-repo_enriched +category = repository +sleep-for-rate = true +no-archive = true +sleep-time = 300 +studies = [enrich_extra_data:github] + +[github:pull] +# Loaded from credentials.cfg +# api-token = +raw_index = github-pull_raw +enriched_index = github-pull_enriched +category = pull_request +sleep-for-rate = true +no-archive = true +studies = [enrich_onion:github,enrich_geolocation:user,enrich_geolocation:assignee,enrich_backlog_analysis,enrich_extra_data:github] + +[github2:issue] +# Loaded from credentials.cfg +# api-token = +raw_index = github2-issue_raw +enriched_index = github2-issue_enriched +sleep-for-rate = true +no-archive = true +category = issue + +[github2:pull] +# Loaded from credentials.cfg +# api-token = +raw_index = github2-pull_raw +enriched_index = gihub2-pull_enriched +sleep-for-rate = true +no-archive = true +category = pull_request + +[discourse] +# Loaded from credentials.cfg +# api-username = +# api-token = +raw_index = discourse-raw +enriched_index = discourse-enriched + +[slack] +# Loaded from credentials.cfg +# api-token = +raw_index = slack-raw +enriched_index = slack-enriched +no-archive = true + +# Studies +# Sample working study to apply labels bases on edX's squad/repo ownership model +[enrich_extra_data:github] +json_url = https://gist.githubusercontent.com/e0d/88669434afa55953b0d99ed04402ee46/raw/2227e48b947c4e822c4888fda8260846e6034d15/json + +[enrich_onion:github] +in_index_iss = github_issues_onion-src +in_index_prs = github_prs_onion-src +out_index_iss = github-issues-onion_enriched +out_index_prs = github-prs-onion_enriched + +[enrich_geolocation:user] +location_field = user_location +geolocation_field = user_geolocation + +[enrich_geolocation:assignee] +location_field = assignee_location +geolocation_field = assignee_geolocation + +[enrich_backlog_analysis] +out_index = github_enrich_backlog +interval_days = 7 +reduced_labels = [bug,enhancement] +map_label = [others, bugs, enhancements] diff --git a/experimental/simple-projects.json b/experimental/simple-projects.json new file mode 100644 index 0000000..2559dea --- /dev/null +++ b/experimental/simple-projects.json @@ -0,0 +1,34 @@ +{ + "Open edX": { + "git": [ + "https://github.com/edx/frontend-app-profile", + "https://github.com/edx/blockstore", + "https://github.com/edx/XBlock", + "https://github.com/edx/configuration" + ], + "github": [ + "https://github.com/edx/frontend-app-profile", + "https://github.com/edx/blockstore", + "https://github.com/edx/XBlock", + "https://github.com/edx/configuration" + ], + "github:repo": [ + "https://github.com/edx/frontend-app-profile", + "https://github.com/edx/blockstore", + "https://github.com/edx/XBlock", + "https://github.com/edx/configuration" + ], + "github:pull": [ + "https://github.com/edx/frontend-app-profile", + "https://github.com/edx/blockstore", + "https://github.com/edx/XBlock", + "https://github.com/edx/configuration" + ], + "slack": [ + "C02SNA1U4" + ], + "discourse": [ + "https://discuss.openedx.org" + ] + } +}