Adapt Twisted Trac ticket migration script (#23)

Worked for Twisted. Still missed references of format ticket:1234 .

* WIP Adding fields to description.

* Add branch_author field. Cleanup.

* Add fields. Reformat output. Remove projects.

* Add status changes, assignee changes.

* Remove CC field for privacy.

* Group changes by time and ticket.

* Add cc__user1 cc__user2 output.

* Fix labels, CC.

* Milestones. Fix some details.

* Fix user mapping, more details.

* Censor e-mails, lowercase labels, more details.

Also linkify changeset hashes and leave alone missing IDs.

* Add branch URLs.

* Filter keywords.

* Fix milestones, wiki links.

* Handle 404s after checking issue too soon.

* Fix missing-comment bug.

* Delete tickets_expected_gold.tsv.twisted-trial-1

README.rst

@@ -65,13 +65,22 @@ Things that are not yet auto-converted:
 
 # Ticket migration
 
-* Copy `config.py.sample` over `config.py`, and edit all the settings.
+The script to use is `ticket_migrate_golden_comet_preview.py`.
+
+* Copy `config.py.sample` to `config.py`, and edit all the settings.
   Perhaps use a fake `OAUTH_TOKEN` to avoid accidental changes.
 * Dump and convert the Postgres DB to SQLite, if you don't already have it,
   using `postgres-to-sqlite.sh`.
-* Get the latest `projects_created.tsv` to avoid duplicating projects.
+  * scp postgres-to-sqlite.sh user@your-server.com:/tmp/postgres-to-sqlite_`date -I`.sh
+  * ssh user@trac-server.com
+  * sudo su trac
+  * cd /tmp
+  * ./postgres-to-sqlite_`date -I`.sh  # Will take ~1 minute for ~10k tickets.
+  * ^D # Exit su trac
+  * ^D # Close SSH
+  * scp user@trac-server.com:/tmp/results.sqlite3 results-`date -I`.sqlite3  # About 70M for 10346 tickets
 * Create required files:
-  `touch tickets_created.tsv && touch tickets_expected_gold.tsv`
+  `touch tickets_created.tsv && touch tickets_expected_gold.tsv && touch milestones_created`
 * Modify `select_tickets` to your liking.
   Perform a dry run, generating `tickets_expected.tsv`.
 * Once the system generated the desired `tickets_expected.tsv`,
@@ -81,11 +90,16 @@ Things that are not yet auto-converted:
   There should be none, if all required tickets are
   in `tickets_expected_gold.tsv`.
 * If you are sure you want to create tickets, change `DRY_RUN` to `False`
-  in `ticket_migrate.py`.
-* Run `./ticket_migrate.py ../trac.db`, where `../trac.db` is the path
+  in `ticket_migrate_golden_comet_preview.py`.
+* Run `python -u ./ticket_migrate_golden_comet_preview.py ../trac.db | tee -a output.txt`, where `../trac.db` is the path
   to the Trac SQLite DB dump.
+* By the first non-dry run breakpoint:
+  * the milestones will have been created. Check `milestones_created.tsv`.
+  * the new `tickets_expected.tsv` must match `tickets_expected_gold.tsv`.
+  * If all is in order, continue by entering `c` at the debugger.
 
-In the event a new ticket is created while the script is running,
+In the event a new ticket or PR is created while the script is running,
 you must manually add a fake entry to `tickets_created.tsv` so that,
 on retrying, as much as possible of `tickets_expected.tsv` still matches
 `tickets_expected_gold.tsv`.
+But you must manually fix any references to that GitHub ID.

config.py.sample

@@ -4,6 +4,17 @@ USER_MAPPING = {
     'danuker': ('danuker', 'Dan Haiduc <danuthaiduc@gmail.com>'),
     }
 
+# GitHub User to GitHub UID mapping (for avatar purposes).
+UID_MAPPING = {
+    'adiroiban': 204609,
+    }
+
+# Users that are allowed by GitHub
+# to be assignees of issues in the target repositories.
+ASSIGNABLE_USERS = {
+    'adiroiban',
+    }
+
 TRAC_TICKET_PREFIX = 'https://trac.chevah.com/ticket/'
 
 # None or a tuple of (name, email)
@@ -37,3 +48,18 @@ ATTACHMENT_ROOT = 'https://raw.githubusercontent.com/twisted/twistedmatrix.com-t
 # https://github.com/settings/tokens
 OAUTH_USER = 'danuker'
 OAUTH_TOKEN = 'ghp_qwertyuiop'
+
+# GitHub User to GitHub UID mapping for avatar and mention purposes.
+UID_MAPPING = {
+    'adiroiban': 204609,
+    'danuker': 74835,
+    }
+
+# E-mails for which not to censor the domain.
+ALLOWED_EMAILS = {
+    'ecdsa-sha2-nistp256-cert-v01@openssh.com',
+    'ecdsa-sha2-nistp384-cert-v01@openssh.com',
+    'ecdsa-sha2-nistp521-cert-v01@openssh.com',
+    'john.doe@example.com',
+    'romeo@montague.net',
+}

config_test.py

@@ -4,19 +4,26 @@
 USER_MAPPING = {
     'adi': ('adiroiban', 'Adi Roiban <adi.roiban@chevah.com>'),
     'danuker': ('danuker', 'Dan Haiduc <danuthaiduc@gmail.com>'),
+    'Adi Roiban': ('adiroiban', 'Adi Roiban <adi.roiban@chevah.com>'),
     }
 
-TRAC_TICKET_PREFIX = 'https://trac.chevah.com/ticket/'
+# GitHub User to GitHub UID mapping (for avatar purposes).
+UID_MAPPING = {
+    'adiroiban': 204609,
+    'mthuurne': 246676,
+    }
 
-# Trac ticket Component to GitHub repository mapping.
-REPOSITORY_MAPPING = {
-    'client': 'client',
-    'commons': 'commons',
-    'trac-migration-staging': 'trac-migration-staging',
+# Users that are allowed by GitHub
+# to be assignees of issues in the target repositories.
+ASSIGNABLE_USERS = {
+    'adiroiban',
     }
 
-# GitHub repository for Trac tickets with Component not in the mapping.
-FALLBACK_REPOSITORY = 'server'
+TRAC_TICKET_PREFIX = 'https://trac.chevah.com/ticket/'
+MIGRATED_WIKI_PREFIX = 'https://example.org/wiki/'
+
+# GitHub repository for Trac tickets.
+REPOSITORY = 'trac-migration-staging'
 
 # Owner of GitHub repositories where to create issues.
 OWNER = 'chevah'
@@ -29,3 +36,15 @@
 # https://github.com/settings/tokens
 OAUTH_USER = 'danuker'
 OAUTH_TOKEN = 'ghp_qwertyuiop'
+
+ALLOWED_EMAILS = {
+    'allowed@example.com',
+    }
+
+ALLOWED_KEYWORDS = {
+    'documentation',
+    'twistd',
+    'review',
+    'windows',
+    'tests',
+    }

postgres-to-sqlite.sh

@@ -4,8 +4,8 @@
 # Adapted from: https://stackoverflow.com/a/56040892
 DBNAME=trac
 
-# ticket_change takes ~5 minutes (140k entries)
-for table_name in attachment ticket ticket_change
+# Export takes ~1 minute
+for table_name in attachment ticket ticket_change ticket_custom session session_attribute milestone
 do
   pg_dump --file "results_dump.sql" --no-password --verbose --format=p --create --clean --disable-dollar-quoting --inserts --column-inserts --section=pre-data --section=data --no-owner  --table "public.${table_name}" $DBNAME