diff --git a/.github/workflows/postinstall.tests.yml b/.github/workflows/postinstall.tests.yml index ec5735a134..9f759ed135 100644 --- a/.github/workflows/postinstall.tests.yml +++ b/.github/workflows/postinstall.tests.yml @@ -32,7 +32,6 @@ on: env: PYTHON3_VERSION: '3.6' - PYTHON2_VERSION: '2.7' jobs: postinstall-tests: @@ -48,11 +47,6 @@ jobs: if: ${{ steps.checkout.outcome == 'success' }} with: python-version: '${{ env.PYTHON3_VERSION }}' - - name: Install Python ${{ env.PYTHON2_VERSION }} - uses: actions/setup-python@v4 - if: ${{ steps.checkout.outcome == 'success' }} - with: - python-version: '${{ env.PYTHON2_VERSION }}' - name: Run Postinstall Tests if: ${{ steps.checkout.outcome == 'success' }} run: traffic_ops/install/bin/postinstall.test.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 1722cd0858..2decbb98f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - [#7918](https://github.com/apache/trafficcontrol/pull/7918) *Traffic Portal* Fixed topology link under DS-Servers tables page - [#7846](https://github.com/apache/trafficcontrol/pull/7846) *Traffic Portal* Increase State character limit +### Removed +- [#7832](https://github.com/apache/trafficcontrol/pull/7832) *t3c* Removed Perl dependency +- [#7841](https://github.com/apache/trafficcontrol/pull/7841) *Postinstall* Removed Perl implementation and Python 2.x support + ## [8.0.0] - 2023-09-20 ### Added - [#7672](https://github.com/apache/trafficcontrol/pull/7672) *Traffic Control Health Client*: Added peer monitor flag while using `strategies.yaml`. diff --git a/docs/source/admin/traffic_ops.rst b/docs/source/admin/traffic_ops.rst index db97fb5580..ce6fdc9a25 100644 --- a/docs/source/admin/traffic_ops.rst +++ b/docs/source/admin/traffic_ops.rst @@ -208,17 +208,20 @@ Guide | Password for the admin user | The password for the administrative Traffic Ops user. | +----------------------------------------------------+------------------------------------------------------------------------------------------------+ -.. deprecated:: ATCv6 - The postinstall script is now written in Python. If you run into issues with the postinstall script, you are encouraged to file an issue at https://github.com/apache/trafficcontrol/issues/new/choose. The original Perl postinstall script is deprecated and will be removed in a future ATC release. To use the deprecated version anyway, run ``/opt/traffic_ops/install/bin/_postinstall.pl`` directly instead of ``/opt/traffic_ops/install/bin/postinstall``. - The postinstall script can also be run non-interactively using :atc-file:`traffic_ops/install/bin/input.json`. To use it, first change the values to match your environment, then pass it to the ``postinstall`` script: .. code-block:: console :caption: Postinstall in Automatic (-a) mode /opt/traffic_ops/install/bin/postinstall -a --cfile /opt/traffic_ops/install/bin/input.json -.. deprecated:: ATCv6 - Once the Perl script is removed, the values in ``input.json`` for the ``"hidden"`` properties will be changed from ``"1"`` and ``"0"`` to ``true`` and ``false``. +.. versionchanged:: ATCv8 + The values in ``input.json`` for the ``"hidden"`` properties have been changed from ``"1"`` and ``"0"`` to ``true`` and ``false``. + +.. versionchanged:: ATCv8 + Python 2.x is no longer supported by the ``postinstall`` script. + +.. versionremoved:: ATCv8 + In earlier versions of ATC, it was possible to run ``postinstall`` using Perl - no longer. .. _to-upgrading: diff --git a/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile index 0c62a00e2e..1a5b001069 100644 --- a/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile +++ b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile @@ -84,7 +84,8 @@ RUN set -o nounset -o errexit -o xtrace && \ perl-libwww-perl \ perl-TermReadKey \ perl-Test-CPAN-Meta \ - perl-WWW-Curl; \ + perl-WWW-Curl \ + python3; \ dnf clean all FROM trafficops-dependencies AS trafficops diff --git a/traffic_ops/build/traffic_ops.spec b/traffic_ops/build/traffic_ops.spec index c2d404c558..536e95074e 100644 --- a/traffic_ops/build/traffic_ops.spec +++ b/traffic_ops/build/traffic_ops.spec @@ -38,7 +38,7 @@ Requires: openssl-devel, perl, perl-core, perl-DBD-Pg, perl-DBI, perl-Di Requires: libidn-devel, libcurl-devel, libcap Requires: postgresql13 >= 13.2 Requires: perl-JSON, perl-libwww-perl, perl-Test-CPAN-Meta, perl-WWW-Curl, perl-TermReadKey, perl-Crypt-ScryptKDF -Requires: python(abi) +Requires: python3 Requires(pre): /usr/sbin/useradd, /usr/bin/getent Requires(postun): /usr/sbin/userdel diff --git a/traffic_ops/install/bin/_postinstall.pl b/traffic_ops/install/bin/_postinstall.pl deleted file mode 100755 index ef15725260..0000000000 --- a/traffic_ops/install/bin/_postinstall.pl +++ /dev/null @@ -1,918 +0,0 @@ -#!/usr/bin/perl - -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -use lib qw(/opt/traffic_ops/install/lib /opt/traffic_ops/app/lib /opt/traffic_ops/app/local/lib/perl5); - -$ENV{PERL5LIB} = "/opt/traffic_ops/install/lib:/opt/traffic_ops/app/lib:/opt/traffic_ops/app/local/lib/perl5:$ENV{PERL5LIB}"; -$ENV{PATH} = "/usr/bin:/usr/local/go/bin:/opt/traffic_ops/install/bin:$ENV{PATH}"; - -use strict; -use warnings; - -use DBI; -use POSIX; -use File::Basename qw{dirname}; -use File::Path qw{make_path}; -use Crypt::ScryptKDF qw(scrypt_hash); -use Data::Dumper qw(Dumper); -use Scalar::Util qw(looks_like_number); -use Getopt::Long; - -use InstallUtils qw{ :all }; -use GenerateCert qw{ :all }; -use Database qw{ connect }; - -# paths of the output configuration files -my $databaseConfFile = "/opt/traffic_ops/app/conf/production/database.conf"; -my $dbConfFile = "/opt/traffic_ops/app/db/dbconf.yml"; -my $cdnConfFile = "/opt/traffic_ops/app/conf/cdn.conf"; -my $ldapConfFile = "/opt/traffic_ops/app/conf/ldap.conf"; -my $usersConfFile = "/opt/traffic_ops/install/data/json/users.json"; -my $profilesConfFile = "/opt/traffic_ops/install/data/profiles/"; -my $opensslConfFile = "/opt/traffic_ops/install/data/json/openssl_configuration.json"; -my $paramConfFile = "/opt/traffic_ops/install/data/json/profiles.json"; - -my $custom_profile_dir = $profilesConfFile . "custom"; - -# stores parameters for traffic ops config -my $parameters; - -# location of traffic ops profiles -my $profileDir = "/opt/traffic_ops/install/data/profiles/"; -my $post_install_cfg = "/opt/traffic_ops/install/data/json/post_install.json"; - -# log file for the installer -my $logFile = "/var/log/traffic_ops/postinstall.log"; - -# debug mode -my $debug = 1; - -# log file for cpan output -my $cpanLogFile = "/var/log/traffic_ops/cpan.log"; - -# maximum size the uncompressed log file should be before rotating it - rotating it copies the current log -# file to the same name appended with .bkp replacing the old backup if any is there -my $maxLogSize = 10000000; #bytes - -# whether to create a config file with default values -my $dumpDefaults; - -# configuration file output with answers which can be used as input to postinstall -my $outputConfigFile = "/opt/traffic_ops/install/bin/configuration_file.json"; - -my $inputFile = ""; -my $automatic = 0; -my %defaultInputs; - -# given a var to the hash of config_var and question, will return the question -sub getConfigQuestion { - my $var = shift; - foreach my $key ( keys %{ $var } ) { - if ( $key ne "hidden" && $key ne "config_var" ) { - return $key; - } - } -} - -# question: The question given in the config file -# config_answer: The answer given in the config file - if no config file given will be defaultInput -# hidden: Whether or not the answer should be hidden from the terminal and logs, ex. passwords -# -# Determines if the script is being run in complete interactive mode and prompts user - otherwise -# returns answer to question in config or defaults - -sub getField { - my $question = shift; - my $config_answer = shift; - my $hidden = shift; - - # if there is no config file and not in automatic mode prompt for all questions with default answers - if ( !$inputFile && !$automatic ) { - - # if hidden then dont show password in terminal - if ($hidden) { - return InstallUtils::promptPasswordVerify($question); - } - else { - return InstallUtils::promptUser( $question, $config_answer ); - } - } - - return $config_answer; -} - -# userInput: The entire input config file which is either user input or the defaults -# fileName: The name of the output config file given by the input config file -# -# Loops through an input config file and determines answers to each question using getField -# and returns the hash of answers - -sub getConfig { - my %userInput = %{$_[0]}; shift; - my $fileName = shift; - - my %config; - - if ( !defined $userInput{$fileName} ) { - InstallUtils::logger( "No $fileName found in config", "error" ); - } - - InstallUtils::logger( "===========$fileName===========", "info" ); - - foreach my $var ( @{ $userInput{$fileName} } ) { - my $question = getConfigQuestion($var); - my $hidden = $var->{"hidden"} if ( exists $var->{"hidden"} ); - my $answer = $config{ $var->{"config_var"} } = getField( $question, $var->{$question}, $hidden ); - - $config{ $var->{"config_var"} } = $answer; - if ( !$hidden ) { - InstallUtils::logger( "$question: $answer", "info" ); - } - } - return %config; -} - -# userInput: The entire input config file which is either user input or the defaults -# dbFileName: The filename of the output config file for the database -# toDBFileName: The filename of the output config file for the Traffic Ops database -# -# Generates a config file for the database based on the questions and answers in the input config file - -sub generateDbConf { - my %userInput = %{$_[0]}; shift; - my $dbFileName = shift; - my $toDBFileName = shift; - - my %dbconf = getConfig( \%userInput, $dbFileName ); - $dbconf{"description"} = "$dbconf{type} database on $dbconf{hostname}:$dbconf{port}"; - make_path( dirname($dbFileName), { mode => 0755 } ); - InstallUtils::writeJson( $dbFileName, \%dbconf ); - InstallUtils::logger( "Database configuration has been saved", "info" ); - - # broken out into separate file/config area - my %todbconf = getConfig( \%userInput, $toDBFileName ); - - # Check if the Postgres db is used and set the driver to be "postgres" - my $dbDriver = $dbconf{type}; - if ( $dbconf{type} eq "Pg" ) { - $dbDriver = "postgres"; - } - - # No YAML library installed, but this is a simple file.. - open( my $fh, '>', $toDBFileName ) or errorOut("Can't write to $toDBFileName!"); - print $fh "production:\n"; - print $fh " driver: $dbDriver\n"; - print $fh " open: host=$dbconf{hostname} port=$dbconf{port} user=$dbconf{user} password=$dbconf{password} dbname=$dbconf{dbname} sslmode=disable\n"; - close $fh; - - return \%todbconf; -} - -# userInput: The entire input config file which is either user input or the defaults -# fileName: The filename of the output config file -# -# Generates a config file for the CDN - -sub generateCdnConf { - my %userInput = %{$_[0]}; shift; - my $fileName = shift; - - my %cdnConfiguration = getConfig( \%userInput, $fileName ); - - # First, read existing one -- already loaded with a bunch of stuff - my $cdnConf; - if ( -f $fileName ) { - $cdnConf = InstallUtils::readJson($fileName) or errorOut("Error loading $fileName: $@"); - } - if ( lc $cdnConfiguration{genSecret} =~ /^y(?:es)?/ ) { - my @secrets; - my $newSecret = InstallUtils::randomWord(); - - if (defined($cdnConf->{secrets})) { - @secrets = @{ $cdnConf->{secrets} }; - $cdnConf->{secrets} = \@secrets; - InstallUtils::logger( "Secrets found in cdn.conf file", "debug" ); - } else { - $cdnConf->{secrets} = \@secrets; - InstallUtils::logger( "No secrets found in cdn.conf file", "debug" ); - } - unshift @secrets, InstallUtils::randomWord(); - if ( $cdnConfiguration{keepSecrets} > 0 && $#secrets > $cdnConfiguration{keepSecrets} - 1 ) { - - # Shorten the array to requested length - $#secrets = $cdnConfiguration{keepSecrets} - 1; - } - } - if (exists $cdnConfiguration{base_url}) { - $cdnConf->{to}{base_url} = $cdnConfiguration{base_url}; - } - if (exists $cdnConfiguration{port}) { - $cdnConf->{"traffic_ops_golang"}{port} = $cdnConfiguration{port}; - } - $cdnConf->{"traffic_ops_golang"}{"log_location_error"} = "/var/log/traffic_ops/error.log"; - $cdnConf->{"traffic_ops_golang"}{"log_location_event"} = "/var/log/traffic_ops/access.log"; - - #InstallUtils::logger("cdnConf: " . Dumper($cdnConf), "info" ); - InstallUtils::writeJson( $fileName, $cdnConf ); - InstallUtils::logger( "CDN configuration has been saved", "info" ); -} - -sub hash_pass { - my $pass = shift; - return scrypt_hash($pass, \64, 16384, 8, 1, 64); -} - -# userInput: The entire input config file which is either user input or the defaults -# fileName: The filename of the output config file -# -# Generates an LDAP config file - -sub generateLdapConf { - my %userInput = %{$_[0]}; shift; - my $fileName = shift; - my %ldapInput = %{@{$userInput{$fileName}}[0]}; - my $useLdap = $ldapInput{"Do you want to set up LDAP?"}; - - if ( !lc $useLdap =~ /^y(?:es)?/ ) { - InstallUtils::logger( "Not setting up ldap", "info" ); - return; - } - - my %ldapConf = getConfig( \%userInput, $fileName ); - # convert any deprecated keys to the correct key name - my %keys_converted = ( password => 'admin_pass', hostname => 'host' ); - for my $key (keys %ldapConf) { - if ( exists $keys_converted{$key} ) { - $ldapConf{ $keys_converted{$key} } = delete $ldapConf{$key}; - } - } - - my @requiredKeys = qw{ host admin_dn admin_pass search_base search_query insecure ldap_timeout_secs }; - for my $k (@requiredKeys) { - if (! exists $ldapConf{$k} ) { - errorOut("$k is a required key in $fileName"); - } - } - - delete $ldapConf{setupLdap}; - - # do a very loose check of form -- 'host' must be hostname:port - if ( $ldapConf{ host } !~ /^\S+:\d+$/ ) { - errorOut("host in $fileName must be of form 'hostname:port'"); - } - - make_path( dirname($fileName), { mode => 0755 } ); - InstallUtils::writeJson( $fileName, \%ldapConf ); -} - -sub generateUsersConf { - my %userInput = %{$_[0]}; shift; - my $fileName = shift; - - my %user = (); - my %config = getConfig( \%userInput, $fileName ); - - $user{username} = $config{tmAdminUser}; - $user{password} = hash_pass( $config{tmAdminPw} ); - - InstallUtils::writeJson( $fileName, \%user ); - $user{password} = $config{tmAdminPw}; - return \%user; -} - -sub generateProfilesDir { - my %userInput = %{$_[0]}; shift; - my $fileName = shift; - - my $userIn = $userInput{$fileName}; -} - -sub generateOpenSSLConf { - my %userInput = %{$_[0]}; shift; - my $fileName = shift; - - my %config = getConfig( \%userInput, $fileName ); - return \%config; -} - -sub generateParamConf { - my %userInput = %{$_[0]}; shift; - my $fileName = shift; - - my %config = getConfig( \%userInput, $fileName ); - InstallUtils::writeJson( $fileName, \%config ); - return \%config; -} - -# check default values for missing config_var parameter -sub sanityCheckDefaults { - foreach my $file ( ( keys %defaultInputs ) ) { - foreach my $defaultValue ( @{ $defaultInputs{$file} } ) { - my $question = getConfigQuestion(\%$defaultValue); - - my %defaultValueHash = %$defaultValue; - if ( !defined $defaultValueHash{"config_var"} - || $defaultValueHash{"config_var"} eq "" ) - { - errorOut("Question '$question' in file '$file' has no config_var"); - } - } - } -} - -# userInput: The entire input config file which is either user input or the defaults -# -# Checks the input config file against the default inputs. If there is a question located in the default inputs which -# is not located in the input config file it will output a warning message. - -sub sanityCheckConfig { - my %userInput = %{$_[0]}; shift; - my $diffs = 0; - - foreach my $file ( ( keys %defaultInputs ) ) { - if ( !defined $userInput{$file} ) { - InstallUtils::logger( "File '$file' found in defaults but not config file", "warn" ); - @{$userInput{$file}} = []; - } - - foreach my $defaultValue ( @{ $defaultInputs{$file} } ) { - - my $found = 0; - foreach my $configValue ( @{ $userInput{$file} } ) { - if ( $defaultValue->{"config_var"} eq $configValue->{"config_var"} ) { - $found = 1; - } - } - - # if the question is not found in the config file add it from defaults - if ( !$found ) { - my $question = getConfigQuestion($defaultValue); - InstallUtils::logger( "Question '$question' found in defaults but not in '$file'", "warn" ); - - my %temp; - my $answer; - my $hidden = exists $defaultValue->{"hidden"} && $defaultValue->{"hidden"} ? 1 : 0; - - # in automatic mode add the missing question with default answer - if ($automatic) { - $answer = $defaultValue->{$question}; - InstallUtils::logger( "Adding question '$question' with default answer " . ( $hidden ? "" : "'$answer'" ), "info" ); - } - - # in interactive mode prompt the user for answer to missing question - else { - InstallUtils::logger( "Prompting user for answer", "info" ); - if ($hidden) { - $answer = InstallUtils::promptPasswordVerify($question); - } - else { - $answer = InstallUtils::promptUser( $question, $defaultValue->{$question} ); - } - } - - %temp = ( - "config_var" => $defaultValue->{"config_var"}, - $question => $answer - ); - - if ($hidden) { - $temp{"hidden"} .= "true"; - } - - push @{ $userInput{$file} }, \%temp; - - $diffs++; - } - } - } - - InstallUtils::logger( "File sanity check complete - found $diffs difference" . ( $diffs == 1 ? "" : "s" ), "info" ); -} - -# A function which returns the default inputs data structure. These questions and answers will be used if there is no -# user input config file or if there are questions in the input config file which do not have answers - -sub getDefaults { - return ( - $databaseConfFile => [ - { - "Database type" => "Pg", - "config_var" => "type" - }, - { - "Database name" => "traffic_ops", - "config_var" => "dbname" - }, - { - "Database server hostname IP or FQDN" => "localhost", - "config_var" => "hostname" - }, - { - "Database port number" => "5432", - "config_var" => "port" - }, - { - "Traffic Ops database user" => "traffic_ops", - "config_var" => "user" - }, - { - "Password for Traffic Ops database user" => "", - "config_var" => "password", - "hidden" => "true" - } - ], - $dbConfFile => [ - { - "Database server root (admin) user" => "postgres", - "config_var" => "pgUser" - }, - { - "Password for database server admin" => "", - "config_var" => "pgPassword", - "hidden" => "true" - } - ], - $cdnConfFile => [ - { - "Generate a new secret?" => "yes", - "config_var" => "genSecret" - }, - { - "Number of secrets to keep?" => "1", - "config_var" => "keepSecrets" - }, - { - "Port to serve on?" => "443", - "config_var" => "port" - }, - { - "Number of workers?" => "12", - "config_var" => "workers" - }, - { - "Traffic Ops url?" => "http://localhost:3000", - "config_var" => "base_url" - }, - { - "ldap.conf location? (default is /opt/traffic_ops/app/conf/ldap.conf)" => "", - "config_var" => "ldap_conf_location" - } - ], - $ldapConfFile => [ - { - "Do you want to set up LDAP?" => "no", - "config_var" => "setupLdap" - }, - { - "LDAP server hostname" => "", - "config_var" => "host" - }, - { - "LDAP Admin DN" => "", - "config_var" => "admin_dn" - }, - { - "LDAP Admin Password" => "", - "config_var" => "admin_pass", - "hidden" => "true" - }, - { - "LDAP Search Base" => "", - "config_var" => "search_base" - }, - { - "LDAP Search Query" => "", - "config_var" => "search_query" - }, - { - "LDAP Skip TLS verify" => "", - "config_var" => "insecure" - }, - { - "LDAP Timeout Seconds" => "", - "config_var" => "ldap_timeout_secs" - } - ], - $usersConfFile => [ - { - "Administration username for Traffic Ops" => "admin", - "config_var" => "tmAdminUser" - }, - { - "Password for the admin user" => "", - "config_var" => "tmAdminPw", - "hidden" => "true" - } - ], - $profilesConfFile => [ - { - "Add custom profiles?" => "no", - "config_var" => "custom_profiles" - } - ], - $opensslConfFile => [ - { - "Do you want to generate a certificate?" => "yes", - "config_var" => "genCert" - }, - { - "Country Name (2 letter code)" => "", - "config_var" => "country" - }, - { - "State or Province Name (full name)" => "", - "config_var" => "state" - }, - { - "Locality Name (eg, city)" => "", - "config_var" => "locality" - }, - { - "Organization Name (eg, company)" => "", - "config_var" => "company" - }, - { - "Organizational Unit Name (eg, section)" => "", - "config_var" => "org_unit" - }, - { - "Common Name (eg, your name or your server's hostname)" => "", - "config_var" => "common_name" - }, - { - "RSA Passphrase" => "CHANGEME!!", - "config_var" => "rsaPassword", - "hidden" => "true" - } - ], - $paramConfFile => [ - { - "Traffic Ops url" => "https://localhost", - "config_var" => "tm.url" - }, - { - "Human-readable CDN Name. (No whitespace, please)" => "kabletown_cdn", - "config_var" => "cdn_name" - }, - { - "DNS sub-domain for which your CDN is authoritative" => "cdn1.kabletown.net", - "config_var" => "dns_subdomain" - } - ], - ); -} - -# carried over from old postinstall -# -# todbconf: The database configuration to be used -# opensslconf: The openssl configuration if any - -sub setupDatabaseData { - my $dbh = shift; - my $adminconf = shift; - my $paramconf = shift; - InstallUtils::logger( "paramconf " . Dumper($paramconf), "info" ); - - my $result; - - my $q = <<"QUERY"; - select exists(select 1 from pg_tables where schemaname = 'public' and tablename = 'tm_user') -QUERY - - my $stmt = $dbh->prepare($q); - $stmt->execute(); - - InstallUtils::logger( "Setting up the database data", "info" ); - my $tables_found; - while ( my $row = $stmt->fetch() ) { - $tables_found = $row->[0]; - } - if ($tables_found) { - InstallUtils::logger( "Found existing tables skipping table creation", "info" ); - } else { - invoke_db_admin_pl("load_schema"); - } - invoke_db_admin_pl("migrate"); - invoke_db_admin_pl("seed"); - invoke_db_admin_pl("patch"); - - # Skip the insert if the admin 'username' is already there. - my $hashed_passwd = hash_pass( $adminconf->{"password"} ); - my $insert_admin = <<"ADMIN"; - insert into tm_user (username, tenant_id, role, local_passwd, confirm_local_passwd) - values ('$adminconf->{"username"}', - (select id from tenant where name = 'root'), - (select id from role where name = 'admin'), - '$hashed_passwd', - '$hashed_passwd' ) - ON CONFLICT (username) DO NOTHING; -ADMIN - $dbh->do($insert_admin); - - insert_cdn($dbh, $paramconf); - insert_parameters($dbh, $paramconf); - insert_profiles($dbh, $paramconf); - - -} - -sub invoke_db_admin_pl { - my $action = shift; - - chdir("/opt/traffic_ops/app"); - my $result = InstallUtils::execCommand( "db/admin", "--env=production", $action ); - - if ( $result != 0 ) { - errorOut("Database $action failed"); - } - else { - InstallUtils::logger( "Database $action succeeded", "info" ); - } - - return $result; -} - -sub setupCertificates { - my $opensslconf = shift; - - my $result; - - if ( lc $opensslconf->{"genCert"} =~ /^y(?:es)?/ ) { - if ( -x "/usr/bin/openssl" ) { - InstallUtils::logger( "Installing SSL Certificates", "info" ); - $result = GenerateCert::createCert($opensslconf); - - if ( $result != 0 ) { - errorOut("SSL Certificate Installation failed"); - } - else { - InstallUtils::logger( "SSL Certificates have been installed", "info" ); - } - } - else { - InstallUtils::logger( "Unable to install SSL certificates as openssl is not installed", "error" ); - InstallUtils::logger( "Install openssl and then run /opt/traffic_ops/install/bin/generateCert to install SSL certificates", "error" ); - exit 4; - } - } - else { - InstallUtils::logger( "Not generating openssl certification", "info" ); - } -} - -#------------------------------------ -sub insert_cdn { - - my $dbh = shift; - my $paramconf = shift; - - InstallUtils::logger( "=========== Setting up cdn", "info" ); - - # Enable multiple inserts into one commit - $dbh->{pg_server_prepare} = 0; - - my $cdn_name = $paramconf->{"cdn_name"}; - my $dns_subdomain = $paramconf->{"dns_subdomain"}; - - my $insert_stmt = <{pg_server_prepare} = 0; - - my $tm_url = $paramconf->{"tm.url"}; - - my $insert_stmt = <{"tm.url"}; - - my $insert_stmt = <prepare($insert_stmt); - $stmt->execute(); -} - - - -# -cfile - Input File: The input config file used to ask and answer questions -# -a - Automatic mode: If there are questions in the config file which do not have answers, the script -# will look to the defaults for the answer. If the answer is not in the defaults -# the script will exit -# -defaults - Defaults: Writes out a configuration file with defaults which can be used as input -# -debug - Debug Mode: More output to the terminal -# -h - Help: Basic command line help menu - -sub main { - my $help = 0; - - # help string - my $usageString = "Usage: postinstall [-a] [-debug] [-defaults[= \$inputFile, - "automatic" => \$automatic, - "defaults:s" => \$dumpDefaults, - "debug" => \$debug, - "help" => \$help - ) or die($usageString); - - # stores the default questions and answers - %defaultInputs = getDefaults(); - - if ($help) { - print $usageString; - return; - } - - # check if the user running postinstall is root - if ( $> != 0 ) { - errorOut("You must run this script as the root user"); - } - - InstallUtils::initLogger( $debug, $logFile ); - - print("unzipping log\n"); - if ( -f "$logFile.gz" ) { - InstallUtils::execCommand( "/bin/gunzip", "-f", "$logFile.gz" ); - } - - InstallUtils::logger( "Starting postinstall", "info" ); - - InstallUtils::logger( "Debug is on", "info" ); - - if ($automatic) { - InstallUtils::logger( "Running in automatic mode", "info" ); - } - - if (defined $dumpDefaults) { - # -defaults flag provided. - if ($dumpDefaults ne "") { - # -defaults= -- if -defaults without a file name, use the default. - # dumpDefaults with value -- use that as output file name - $outputConfigFile = $dumpDefaults; - } - InstallUtils::logger( "Writing default configuration to $outputConfigFile", "info" ); - InstallUtils::writeJson( $outputConfigFile, %defaultInputs ); - return; - } - - InstallUtils::rotateLog($cpanLogFile); - - if ( -s $logFile > $maxLogSize ) { - InstallUtils::logger( "Postinstall log above max size of $maxLogSize bytes - rotating", "info" ); - rotateLog($logFile); - } - - # used to store the questions and answers provided by the user - my %userInput; - - # if no input file provided use the defaults - if ( $inputFile eq "" ) { - InstallUtils::logger( "No input file given - using defaults", "info" ); - %userInput = %defaultInputs; - } - else { - InstallUtils::logger( "Using input file $inputFile", "info" ); - - # check if the input file exists - errorOut("File '$inputFile' not found") if ( !-f $inputFile ); - - # read and store the input file - %userInput = %{InstallUtils::readJson($inputFile)}; - } - - # sanity check the defaults if running them automatically - sanityCheckDefaults(); - - # check the input config file against the defaults to check for missing questions - sanityCheckConfig(\%userInput) if ( $inputFile ne "" ); - - chdir("/opt/traffic_ops/install/bin"); - - # The generator functions handle checking input/default/automatic mode - # todbconf will be used later when setting up the database - my $todbconf = generateDbConf( \%userInput, $databaseConfFile, $dbConfFile ); - generateLdapConf( \%userInput, $ldapConfFile ); - my $adminconf = generateUsersConf( \%userInput, $usersConfFile ); - my $custom_profile = generateProfilesDir( \%userInput, $profilesConfFile ); - my $opensslconf = generateOpenSSLConf( \%userInput, $opensslConfFile ); - my $paramconf = generateParamConf( \%userInput, $paramConfFile ); - - if ( !-f $post_install_cfg ) { - InstallUtils::writeJson( $post_install_cfg, {} ); - } - - setupCertificates( $opensslconf ); - generateCdnConf( \%userInput, $cdnConfFile ); - - my $dbh = Database::connect($databaseConfFile, $todbconf); - if (!$dbh) { - InstallUtils::logger("Can't connect to the database. Use the script `/opt/traffic_ops/install/bin/todb_bootstrap.sh` on the db server to create it and run `postinstall` again.", "error"); - exit(-1); - } - - setupDatabaseData( $dbh, $adminconf, $paramconf ); - - InstallUtils::logger("Starting Traffic Ops", "info" ); - InstallUtils::execCommand("/sbin/service traffic_ops restart"); - - InstallUtils::logger("Waiting for Traffic Ops to restart", "info" ); - - InstallUtils::logger("Success! Postinstall complete."); - - #InstallUtils::logger("Zipping up $logFile to $logFile.gz"); - #InstallUtils::execCommand( "/bin/gzip", "$logFile" ); - - # Success! - $dbh->disconnect(); -} - -main; - -# vi:syntax=perl diff --git a/traffic_ops/install/bin/_postinstall.py b/traffic_ops/install/bin/_postinstall.py index 192eaf516a..62c5309360 100755 --- a/traffic_ops/install/bin/_postinstall.py +++ b/traffic_ops/install/bin/_postinstall.py @@ -1,23 +1,4 @@ #!/usr/bin/env python3 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# There's a bug in asteroid with Python 3.9's NamedTuple being -# recognized for the dynamically generated class that it is. Should be fixed -# with the next release, but 'til then... -#pylint:disable=inherit-non-class -from __future__ import print_function - """ This script is meant as a drop-in replacement for the old _postinstall.pl Perl script. @@ -45,6 +26,23 @@ >>> [c for c in [[a for a in b if not a.config_var] for b in DEFAULTS.values()] if c] [] """ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# There's a bug in asteroid with Python 3.9's NamedTuple being +# recognized for the dynamically generated class that it is. Should be fixed +# with the next release, but 'til then... + import argparse import base64 import errno @@ -63,8 +61,8 @@ import subprocess import sys -from collections import namedtuple from struct import unpack, pack +from typing import Any, Dict, List, Optional, Union, NamedTuple # Paths for output configuration files DATABASE_CONF_FILE = "/opt/traffic_ops/app/conf/production/database.conf" @@ -95,14 +93,9 @@ # Python, instead, outputs to stdout. This is breaking, but more flexible. Change it? # OUTPUT_CONFIG_FILE = "/opt/traffic_ops/install/bin/configuration_file.json" -if sys.version_info.major >= 3: - # Accepting a string for json.dump()'s `indent` keyword argument is a Python 3 feature - indent = "\t" # type: str -else: - indent = 4 # type: int - str = unicode # type: type[unicode] +INDENT = "\t" -class Question(object): +class Question: """ Question represents a single question to be asked of the user, to determine a configuration value. @@ -111,25 +104,25 @@ class Question(object): Question(question='question', default='answer', config_var='var', hidden=False) """ - def __init__(self, question, default, config_var, hidden = False): # type: (str, str, str, bool) -> None + def __init__(self, question: str, default: str, config_var: str, hidden: bool = False) -> None: self.question = question self.default = default self.config_var = config_var self.hidden = hidden - def __str__(self): # type: () -> str + def __str__(self) -> str: if self.default: - return "{question} [{default}]: ".format(question=self.question, default=self.default) - return "{question}: ".format(question=self.question) + return f"{self.question} [{self.default}]: " + return f"{self.question}: " - def __repr__(self): # type: () -> str + def __repr__(self) -> str: qstn = self.question ans = self.default cfgvr = self.config_var hddn = self.hidden - return "Question(question='{qstn}', default='{ans}', config_var='{cfgvr}', hidden={hddn})".format(qstn=qstn, ans=ans, cfgvr=cfgvr, hddn=hddn) + return f"Question(question='{qstn}', default='{ans}', config_var='{cfgvr}', hidden={hddn})" - def ask(self): # type: () -> str + def ask(self) -> str: """ Asks the user the Question interactively. @@ -140,13 +133,13 @@ def ask(self): # type: () -> str passwd = getpass.getpass(str(self)) if not passwd: continue - if passwd == getpass.getpass("Re-Enter {question}: ".format(question=self.question)): + if passwd == getpass.getpass(f"Re-Enter {self.question}: "): return passwd print("Error: passwords do not match, try again") ipt = input(self) return ipt if ipt else self.default - def to_json(self): # type: () -> str + def to_json(self) -> str: """ Converts a question to JSON encoding. @@ -159,15 +152,16 @@ def to_json(self): # type: () -> str ans = self.default cfgvr = self.config_var if self.hidden: - return '{{"{qstn}": "{ans}", "config_var": "{cfgvr}", "hidden": true}}'.format(qstn=qstn, ans=ans, cfgvr=cfgvr) - return '{{"{qstn}": "{ans}", "config_var": "{cfgvr}"}}'.format(qstn=qstn, ans=ans, cfgvr=cfgvr) + return f'{{"{qstn}": "{ans}", "config_var": "{cfgvr}", "hidden": true}}' + return f'{{"{qstn}": "{ans}", "config_var": "{cfgvr}"}}' - def serialize(self): # type: () -> object + def serialize(self) -> Dict[str, Union[str, bool]]: """Returns a serializable dictionary, suitable for converting to JSON.""" return {self.question: self.default, "config_var": self.config_var, "hidden": self.hidden} -class User(namedtuple('User', ['username', 'password'])): - """Users represents a user that will be inserted into the Traffic Ops database. +class User(NamedTuple): + """ + A User represents a user that will be inserted into the Traffic Ops database. Attributes ---------- @@ -176,21 +170,29 @@ class User(namedtuple('User', ['username', 'password'])): self.password: str The user's password - IN PLAINTEXT. """ + username: str + password: str class SSLConfig: """SSLConfig bundles the options for generating new (self-signed) SSL certificates""" - def __init__(self, gen_cert, cfg_map): # type: (bool, dict[str, str]) -> None - + def __init__(self, gen_cert: bool, cfg_map: Dict[str, str]) -> None: self.gen_cert = gen_cert self.rsa_password = cfg_map["rsaPassword"] self.params = "/C={country}/ST={state}/L={locality}/O={company}/OU={org_unit}/CN={common_name}/" self.params = self.params.format(**cfg_map) -class CDNConfig(namedtuple('CDNConfig', ['gen_secret', 'num_secrets', 'port', 'num_workers', 'url', 'ldap_conf_location'])): +class CDNConfig(NamedTuple): """CDNConfig holds all of the options needed to format a cdn.conf file.""" - def generate_secret(self, conf): + gen_secret: bool + num_secrets: int + port: str + num_workers: int + url: str + ldap_conf_location: str + + def generate_secret(self, conf: Dict[Any, Any]): """ Generates new secrets - if configured to do so - and adds them to the passed cdn.conf configuration. @@ -198,18 +200,19 @@ def generate_secret(self, conf): if not self.gen_secret: return - if isinstance(conf, dict) and "secrets" in conf and isinstance(conf["secrets"], list): + if "secrets" in conf and isinstance(conf["secrets"], list): logging.debug("Secrets found in cdn.conf file") else: conf["secrets"] = [] logging.debug("No secrets found in cdn.conf file") - conf["secrets"].insert(0, random_word()) + secrets: List[str] = conf["secrets"] + secrets.insert(0, random_word()) - if self.num_secrets and len(conf["secrets"]) > self.num_secrets: - conf["secrets"] = conf["secrets"][:self.num_secrets - 1] + if self.num_secrets and len(secrets) > self.num_secrets: + conf["secrets"] = secrets[:self.num_secrets - 1] - def insert_url(self, conf): + def insert_url(self, conf: Dict[Any, Any]): """ Inserts the configured URL - if it is not an empty string - into the passed cdn.conf configuration, in to.base_url. @@ -293,8 +296,7 @@ class ConfigEncoder(json.JSONEncoder): '{"/test/file": [{"question": "default", "config_var": "cfg_var", "hidden": true}]}' """ - # The linter is just wrong about this - def default(self, o): # type: (object) -> object + def default(self, o: Any) -> Any: """ Returns a serializable representation of 'o'. @@ -307,12 +309,12 @@ def default(self, o): # type: (object) -> object return json.JSONEncoder.default(self, o) -def get_config(questions, fname, automatic = False): # type: (list[Question], str, bool) -> dict[str, str] +def get_config(questions: List[Question], fname: str, automatic: bool = False) -> Dict[str, str]: """Asks all provided questions, or uses their defaults in automatic mode""" logging.info("===========%s===========", fname) - config = {} + config: Dict[str, str] = {} for question in questions: answer = question.default if automatic else question.ask() @@ -321,7 +323,7 @@ def get_config(questions, fname, automatic = False): # type: (list[Question], st return config -def generate_db_conf(qstns, fname, automatic, root): # (list[Question], str, bool, str) -> dict +def generate_db_conf(qstns: List[Question], fname: str, automatic: bool, root: str) -> Dict[str, str]: """ Generates the database.conf file and returns a map of its configuration. @@ -332,18 +334,18 @@ def generate_db_conf(qstns, fname, automatic, root): # (list[Question], str, boo hostname = db_conf.get("hostname", "UNKNOWN") port = db_conf.get("port", "UNKNOWN") - db_conf["description"] = "{typ} database on {hostname}:{port}".format(typ=typ, hostname=hostname, port=port) + db_conf["description"] = f"{typ} database on {hostname}:{port}" path = os.path.join(root, fname.lstrip('/')) - with open(path, 'w+') as conf_file: - json.dump(db_conf, conf_file, indent=indent) + with open(path, 'w+', encoding="utf-8") as conf_file: + json.dump(db_conf, conf_file, indent=INDENT) print(file=conf_file) logging.info("Database configuration has been saved") return db_conf -def generate_todb_conf(fname, root, conf): # (str, str, dict) +def generate_todb_conf(fname: str, root: str, conf: Dict[str, str]): """ Generates the dbconf.yml file. @@ -356,20 +358,20 @@ def generate_todb_conf(fname, root, conf): # (str, str, dict) else: driver = "postgres" if conf["type"] == "Pg" else conf["type"] - path = os.path.join(root, fname.lstrip('/')) - hostname = conf.get('hostname', 'UNKNOWN') - port = conf.get('port', 'UNKNOWN') - user = conf.get('user', 'UNKNOWN') - password = conf.get('password', 'UNKNOWN') - dbname = conf.get('dbname', 'UNKNOWN') - - open_line = "host={hostname} port={port} user={user} password={password} dbname={dbname}".format(hostname=hostname, port=port, user=user, password=password, dbname=dbname) - with open(path, 'w+') as conf_file: + path = os.path.join(root, fname.lstrip("/")) + hostname = conf.get("hostname", "UNKNOWN") + port = conf.get("port", "UNKNOWN") + user = conf.get("user", "UNKNOWN") + password = conf.get("password", "UNKNOWN") + dbname = conf.get("dbname", "UNKNOWN") + + open_line = f"host={hostname} port={port} user={user} password={password} dbname={dbname}" + with open(path, "w+", encoding="utf-8") as conf_file: print("production:", file=conf_file) print(" driver:", driver, file=conf_file) - print(" open: {open_line} sslmode=disable".format(open_line=open_line), file=conf_file) + print(" open:", open_line, "sslmode=disable", file=conf_file) -def generate_ldap_conf(questions, fname, automatic, root): # type: (list[Question], str, bool, str) -> None +def generate_ldap_conf(questions: List[Question], fname: str, automatic: bool, root: str): """ Generates the ldap.conf file by asking the questions or using default answers in auto mode. @@ -381,32 +383,32 @@ def generate_ldap_conf(questions, fname, automatic, root): # type: (list[Questio return use_ldap = use_ldap_question[0].default if automatic else use_ldap_question[0].ask() - if use_ldap.lower() not in {'y', 'yes'}: + if use_ldap.lower() not in {"y", "yes"}: logging.info("Not setting up ldap") return ldap_conf = get_config([q for q in questions if q is not use_ldap_question[0]], fname, automatic) keys = ( - 'host', - 'admin_dn', - 'admin_pass', - 'search_base', - 'search_query', - 'insecure', - 'ldap_timeout_secs' + "host", + "admin_dn", + "admin_pass", + "search_base", + "search_query", + "insecure", + "ldap_timeout_secs" ) for key in keys: if key not in ldap_conf: - raise ValueError("{key} is a required key in {fname}".format(key=key, fname=fname)) + raise ValueError(f"{key} is a required key in {fname}") - keys_converted = {'password': 'admin_pass', 'hostname': 'host'} + keys_converted = {"password": "admin_pass", "hostname": "host"} for deprecated, key in keys_converted.items(): if deprecated in ldap_conf and ldap_conf[key] == '': ldap_conf[key] = ldap_conf[deprecated] if not re.match(r"^\S+:\d+$", ldap_conf["host"]): - raise ValueError("host in {fname} must be of form 'hostname:port'".format(fname=fname)) + raise ValueError(f"host in {fname} must be of form 'hostname:port'") path = os.path.join(root, fname.lstrip('/')) try: @@ -414,11 +416,11 @@ def generate_ldap_conf(questions, fname, automatic, root): # type: (list[Questio except OSError as e: if e.errno == errno.EEXIST: pass - with open(path, 'w+') as conf_file: - json.dump(ldap_conf, conf_file, indent=indent) + with open(path, "w+", encoding="utf-8") as conf_file: + json.dump(ldap_conf, conf_file, indent=INDENT) print(file=conf_file) -def hash_pass(passwd): # type: (str) -> str +def hash_pass(passwd: str) -> str: """ Generates a Scrypt-based hash of the given password in a Perl-compatible format. It's hard-coded - like the Perl - to use 64 random bytes for the salt, n=16384, @@ -436,138 +438,144 @@ def hash_pass(passwd): # type: (str) -> str hashed_b64 = base64.standard_b64encode(hashed).decode() salt_b64 = base64.standard_b64encode(salt).decode() - return "SCRYPT:{n}:{r_val}:{p_val}:{salt_b64}:{hashed_b64}".format(n=n, r_val=r_val, p_val=p_val, salt_b64=salt_b64, hashed_b64=hashed_b64) - + return f"SCRYPT:{n}:{r_val}:{p_val}:{salt_b64}:{hashed_b64}" class Scrypt: - def __init__(self, password, salt, cost_factor, block_size_factor, parallelization_factor, key_length): # type: (bytes, bytes, int, int, int, int) -> None - self.password = password # type: bytes - self.salt = salt # type: bytes - self.cost_factor = cost_factor # type: int - self.block_size_factor = block_size_factor # type: int - self.parallelization_factor = parallelization_factor # type: int + """ + Implements SCRYPT encryption based on the configuration given at object + construction. + """ + def __init__(self, password: bytes, salt: bytes, cost_factor: int, block_size_factor: int, parallelization_factor: int, key_length: int): + self.password = password + self.salt = salt + self.cost_factor = cost_factor + self.block_size_factor = block_size_factor + self.parallelization_factor = parallelization_factor self.key_length = key_length - self.block_unit = 32 * self.block_size_factor # 1 block unit = 32 * block_size_factor 32-bit ints + self.block_unit = 32 * self.block_size_factor - def derive(self): # type: () -> bytes - salt_length = 2 ** 7 * self.block_size_factor * self.parallelization_factor # type: int - pack_format = '<' + 'L' * int(salt_length / 4) # `<` means `little-endian` and `L` means `unsigned long` - salt = hashlib.pbkdf2_hmac('sha256', password=self.password, salt=self.salt, iterations=1, dklen=salt_length) # type: bytes - block = list(unpack(pack_format, salt)) # type: list[int] + def derive(self) -> bytes: + """ + Derives an encrypted bytestring representative of the password given at + initialization. + """ + salt_length = 2 ** 7 * self.block_size_factor * self.parallelization_factor + pack_format = f"<{'L' * int(salt_length / 4)}" # `<` means `little-endian` and `L` means `unsigned long` + salt = hashlib.pbkdf2_hmac('sha256', password=self.password, salt=self.salt, iterations=1, dklen=salt_length) + block = list(unpack(pack_format, salt)) block = self.ROMix(block) salt = pack(pack_format, *block) - key = hashlib.pbkdf2_hmac('sha256', password=self.password, salt=salt, iterations=1, dklen=self.key_length) # type: bytes + key = hashlib.pbkdf2_hmac('sha256', password=self.password, salt=salt, iterations=1, dklen=self.key_length) return key - def ROMix(self, block): # type: (list[int]) -> list[int] - xored_block = [0] * len(block) # type: list[int] - variations = [list()] * self.cost_factor # type: list[list[int]] + def ROMix(self, block: List[int]) -> List[int]: + xored_block = [0] * len(block) + variations: List[List[int]] = [[]] * self.cost_factor variations[0] = block index = 1 while index < self.cost_factor: variations[index] = self.block_mix(variations[index - 1]) index += 1 block = self.block_mix(variations[-1]) - for unused in variations: - variation_index = block[self.block_unit - 16] % self.cost_factor # type: int + for _ in variations: + variation_index = block[self.block_unit - 16] % self.cost_factor variation = variations[variation_index] - for index, unused in enumerate(xored_block): + for index, _unused in enumerate(xored_block): xored_block[index] = block[index] ^ variation[index] block = self.block_mix(xored_block) return block - def block_mix(self, previous_block): # type: (list[int]) -> list[int] - block = previous_block[:] # type: list[int] - X_length = 16 # X is the list of numbers within `block` that we mix - copy_index = self.block_unit - X_length - X = previous_block[copy_index:copy_index + X_length] # type: list[int] - octet_index = 0 # type: int + def block_mix(self, previous_block: List[int]) -> List[int]: + block = previous_block.copy() + x_length = 16 # x is the list of numbers within `block` that we mix + copy_index = self.block_unit - x_length + x = previous_block[copy_index:copy_index + x_length] + octet_index = 0 block_xor_index = 0 while octet_index < 2 * self.block_size_factor: - for index, unused in enumerate(X): - X[index] ^= previous_block[block_xor_index + index] - block_xor_index += X_length - self.salsa20(X) - block_offset = (int(octet_index / 2) + octet_index % 2 * self.block_size_factor) * X_length - block[block_offset:block_offset + X_length] = X + for index, _ in enumerate(x): + x[index] ^= previous_block[block_xor_index + index] + block_xor_index += x_length + self.salsa20(x) + block_offset = (int(octet_index / 2) + octet_index % 2 * self.block_size_factor) * x_length + block[block_offset:block_offset + x_length] = x octet_index += 1 return block - def salsa20(self, block): # type: (list[int]) -> None - X = block[:] # make a copy (list.copy() is Python 3-only) - for i in range(0, 4): + def salsa20(self, block: List[int]): + x = block.copy() + for _ in range(4): # These bit shifting operations could be condensed into a single line of list comprehensions, # but there is a >3x performance benefit from writing it out explicitly. - bits = X[0] + X[12] & 0xffffffff - X[4] ^= bits << 7 | bits >> 32 - 7 - bits = X[4] + X[0] & 0xffffffff - X[8] ^= bits << 9 | bits >> 32 - 9 - bits = X[8] + X[4] & 0xffffffff - X[12] ^= bits << 13 | bits >> 32 - 13 - bits = X[12] + X[8] & 0xffffffff - X[0] ^= bits << 18 | bits >> 32 - 18 - bits = X[5] + X[1] & 0xffffffff - X[9] ^= bits << 7 | bits >> 32 - 7 - bits = X[9] + X[5] & 0xffffffff - X[13] ^= bits << 9 | bits >> 32 - 9 - bits = X[13] + X[9] & 0xffffffff - X[1] ^= bits << 13 | bits >> 32 - 13 - bits = X[1] + X[13] & 0xffffffff - X[5] ^= bits << 18 | bits >> 32 - 18 - bits = X[10] + X[6] & 0xffffffff - X[14] ^= bits << 7 | bits >> 32 - 7 - bits = X[14] + X[10] & 0xffffffff - X[2] ^= bits << 9 | bits >> 32 - 9 - bits = X[2] + X[14] & 0xffffffff - X[6] ^= bits << 13 | bits >> 32 - 13 - bits = X[6] + X[2] & 0xffffffff - X[10] ^= bits << 18 | bits >> 32 - 18 - bits = X[15] + X[11] & 0xffffffff - X[3] ^= bits << 7 | bits >> 32 - 7 - bits = X[3] + X[15] & 0xffffffff - X[7] ^= bits << 9 | bits >> 32 - 9 - bits = X[7] + X[3] & 0xffffffff - X[11] ^= bits << 13 | bits >> 32 - 13 - bits = X[11] + X[7] & 0xffffffff - X[15] ^= bits << 18 | bits >> 32 - 18 - bits = X[0] + X[3] & 0xffffffff - X[1] ^= bits << 7 | bits >> 32 - 7 - bits = X[1] + X[0] & 0xffffffff - X[2] ^= bits << 9 | bits >> 32 - 9 - bits = X[2] + X[1] & 0xffffffff - X[3] ^= bits << 13 | bits >> 32 - 13 - bits = X[3] + X[2] & 0xffffffff - X[0] ^= bits << 18 | bits >> 32 - 18 - bits = X[5] + X[4] & 0xffffffff - X[6] ^= bits << 7 | bits >> 32 - 7 - bits = X[6] + X[5] & 0xffffffff - X[7] ^= bits << 9 | bits >> 32 - 9 - bits = X[7] + X[6] & 0xffffffff - X[4] ^= bits << 13 | bits >> 32 - 13 - bits = X[4] + X[7] & 0xffffffff - X[5] ^= bits << 18 | bits >> 32 - 18 - bits = X[10] + X[9] & 0xffffffff - X[11] ^= bits << 7 | bits >> 32 - 7 - bits = X[11] + X[10] & 0xffffffff - X[8] ^= bits << 9 | bits >> 32 - 9 - bits = X[8] + X[11] & 0xffffffff - X[9] ^= bits << 13 | bits >> 32 - 13 - bits = X[9] + X[8] & 0xffffffff - X[10] ^= bits << 18 | bits >> 32 - 18 - bits = X[15] + X[14] & 0xffffffff - X[12] ^= bits << 7 | bits >> 32 - 7 - bits = X[12] + X[15] & 0xffffffff - X[13] ^= bits << 9 | bits >> 32 - 9 - bits = X[13] + X[12] & 0xffffffff - X[14] ^= bits << 13 | bits >> 32 - 13 - bits = X[14] + X[13] & 0xffffffff - X[15] ^= bits << 18 | bits >> 32 - 18 - - for index in range(0, 16): - block[index] = block[index] + X[index] & 0xffffffff - - -def generate_users_conf(qstns, fname, auto, root): # type: (list[Question], str, bool, str) -> User + bits = x[0] + x[12] & 0xffffffff + x[4] ^= bits << 7 | bits >> 32 - 7 + bits = x[4] + x[0] & 0xffffffff + x[8] ^= bits << 9 | bits >> 32 - 9 + bits = x[8] + x[4] & 0xffffffff + x[12] ^= bits << 13 | bits >> 32 - 13 + bits = x[12] + x[8] & 0xffffffff + x[0] ^= bits << 18 | bits >> 32 - 18 + bits = x[5] + x[1] & 0xffffffff + x[9] ^= bits << 7 | bits >> 32 - 7 + bits = x[9] + x[5] & 0xffffffff + x[13] ^= bits << 9 | bits >> 32 - 9 + bits = x[13] + x[9] & 0xffffffff + x[1] ^= bits << 13 | bits >> 32 - 13 + bits = x[1] + x[13] & 0xffffffff + x[5] ^= bits << 18 | bits >> 32 - 18 + bits = x[10] + x[6] & 0xffffffff + x[14] ^= bits << 7 | bits >> 32 - 7 + bits = x[14] + x[10] & 0xffffffff + x[2] ^= bits << 9 | bits >> 32 - 9 + bits = x[2] + x[14] & 0xffffffff + x[6] ^= bits << 13 | bits >> 32 - 13 + bits = x[6] + x[2] & 0xffffffff + x[10] ^= bits << 18 | bits >> 32 - 18 + bits = x[15] + x[11] & 0xffffffff + x[3] ^= bits << 7 | bits >> 32 - 7 + bits = x[3] + x[15] & 0xffffffff + x[7] ^= bits << 9 | bits >> 32 - 9 + bits = x[7] + x[3] & 0xffffffff + x[11] ^= bits << 13 | bits >> 32 - 13 + bits = x[11] + x[7] & 0xffffffff + x[15] ^= bits << 18 | bits >> 32 - 18 + bits = x[0] + x[3] & 0xffffffff + x[1] ^= bits << 7 | bits >> 32 - 7 + bits = x[1] + x[0] & 0xffffffff + x[2] ^= bits << 9 | bits >> 32 - 9 + bits = x[2] + x[1] & 0xffffffff + x[3] ^= bits << 13 | bits >> 32 - 13 + bits = x[3] + x[2] & 0xffffffff + x[0] ^= bits << 18 | bits >> 32 - 18 + bits = x[5] + x[4] & 0xffffffff + x[6] ^= bits << 7 | bits >> 32 - 7 + bits = x[6] + x[5] & 0xffffffff + x[7] ^= bits << 9 | bits >> 32 - 9 + bits = x[7] + x[6] & 0xffffffff + x[4] ^= bits << 13 | bits >> 32 - 13 + bits = x[4] + x[7] & 0xffffffff + x[5] ^= bits << 18 | bits >> 32 - 18 + bits = x[10] + x[9] & 0xffffffff + x[11] ^= bits << 7 | bits >> 32 - 7 + bits = x[11] + x[10] & 0xffffffff + x[8] ^= bits << 9 | bits >> 32 - 9 + bits = x[8] + x[11] & 0xffffffff + x[9] ^= bits << 13 | bits >> 32 - 13 + bits = x[9] + x[8] & 0xffffffff + x[10] ^= bits << 18 | bits >> 32 - 18 + bits = x[15] + x[14] & 0xffffffff + x[12] ^= bits << 7 | bits >> 32 - 7 + bits = x[12] + x[15] & 0xffffffff + x[13] ^= bits << 9 | bits >> 32 - 9 + bits = x[13] + x[12] & 0xffffffff + x[14] ^= bits << 13 | bits >> 32 - 13 + bits = x[14] + x[13] & 0xffffffff + x[15] ^= bits << 18 | bits >> 32 - 18 + + for index in range(16): + block[index] = block[index] + x[index] & 0xffffffff + +def generate_users_conf(qstns: List[Question], fname: str, auto: bool, root: str) -> User: """ Generates a users.json file from the given questions and returns a User containing the same information. @@ -575,18 +583,18 @@ def generate_users_conf(qstns, fname, auto, root): # type: (list[Question], str, config = get_config(qstns, fname, auto) if "tmAdminUser" not in config or "tmAdminPw" not in config: - raise ValueError("{fname} must include 'tmAdminUser' and 'tmAdminPw'".format(fname=fname)) + raise ValueError(f"{fname} must include 'tmAdminUser' and 'tmAdminPw'") hashed_pass = hash_pass(config["tmAdminPw"]) path = os.path.join(root, fname.lstrip('/')) - with open(path, 'w+') as conf_file: - json.dump({"username": config["tmAdminUser"], "password": hashed_pass}, conf_file, indent=indent) + with open(path, "w+", encoding="utf-8") as conf_file: + json.dump({"username": config["tmAdminUser"], "password": hashed_pass}, conf_file, indent=INDENT) print(file=conf_file) return User(config["tmAdminUser"], config["tmAdminPw"]) -def generate_profiles_dir(questions): # type: (list[Question]) -> None +def generate_profiles_dir(questions: List[Question]): """ I truly have no idea what's going on here. This is what the Perl did, so I copied it. It does nothing. Literally nothing. @@ -595,7 +603,7 @@ def generate_profiles_dir(questions): # type: (list[Question]) -> None user_in = questions #pylint:enable=unused-variable -def generate_openssl_conf(questions, fname, auto): # type: (list[Question], str, bool) -> SSLConfig +def generate_openssl_conf(questions: List[Question], fname: str, auto: bool) -> SSLConfig: """ Constructs an SSLConfig by asking the passed questions, or using their default answers if in auto mode. @@ -608,7 +616,7 @@ def generate_openssl_conf(questions, fname, auto): # type: (list[Question], str, return SSLConfig(gen_cert, cfg_map) -def generate_param_conf(qstns, fname, auto, root): # type: (list[Question], str, bool, str) -> dict +def generate_param_conf(qstns: List[Question], fname: str, auto: bool, root: str) -> Dict[str, str]: """ Generates a profiles.json by asking the passed questions, or using their default answers in auto mode. @@ -617,14 +625,14 @@ def generate_param_conf(qstns, fname, auto, root): # type: (list[Question], str, """ conf = get_config(qstns, fname, auto) - path = os.path.join(root, fname.lstrip('/')) - with open(path, 'w+') as conf_file: - json.dump(conf, conf_file, indent=indent) + path = os.path.join(root, fname.lstrip("/")) + with open(path, "w+", encoding="utf-8") as conf_file: + json.dump(conf, conf_file, indent=INDENT) print(file=conf_file) return conf -def sanity_check_config(cfg, automatic): # type: (dict[str, list[Question]], bool) -> int +def sanity_check_config(cfg: Dict[str, List[Question]], automatic: bool) -> int: """ Checks a user-input configuration file, and outputs the number of files in the default question set that did not appear in the input. @@ -664,7 +672,7 @@ def sanity_check_config(cfg, automatic): # type: (dict[str, list[Question]], boo return diffs -def unmarshal_config(dct): # type: (dict) -> dict[str, list[Question]] +def unmarshal_config(dct: Dict[str, Any]) -> Dict[str, List[Question]]: """ Reads in a raw parsed configuration file and returns the resulting configuration. @@ -673,23 +681,25 @@ def unmarshal_config(dct): # type: (dict) -> dict[str, list[Question]] >>> unmarshal_config({"test": [{"foo": "", "config_var": "bar", "hidden": True}]}) {'test': [Question(question='foo', default='', config_var='bar', hidden=True)]} """ - ret = {} + ret: Dict[str, List[Question]] = {} for file, questions in dct.items(): if not isinstance(questions, list): - raise ValueError("file '{file}' has malformed questions".format(file=file)) + raise ValueError(f"file '{file}' has malformed questions") - qstns = [] + qstns: List[Question] = [] + qstn: Any for qstn in questions: if not isinstance(qstn, dict): - raise ValueError("file '{file}' has a malformed question ({qstn})".format(file=file, qstn=qstn)) + raise ValueError(f"file '{file}' has a malformed question ({qstn})") + question: Any try: - question = next(key for key in qstn.keys() if key not in ("hidden", "config_var")) + question = next(key for key in qstn.keys() if key not in {"hidden", "config_var"}) except StopIteration: - raise ValueError("question in '{file}' has no question/answer properties ({qstn})".format(file=file, qstn=qstn)) + raise ValueError(f"question in '{file}' has no question/answer properties ({qstn})") - answer = qstn[question] + answer: Any = qstn[question] if not isinstance(question, str) or not isinstance(answer, str): - errstr = "question in '{file}' has malformed question/answer property ({question}: {answer})".format(file=file, question=question, answer=answer) + errstr = f"question in '{file}' has malformed question/answer property ({question}: {answer})" raise ValueError(errstr) del qstn[question] @@ -699,10 +709,10 @@ def unmarshal_config(dct): # type: (dict) -> dict[str, list[Question]] del qstn["hidden"] if "config_var" not in qstn: - raise ValueError("question in '{file}' has no 'config_var' property".format(file=file)) - cfg_var = qstn["config_var"] + raise ValueError(f"question in '{file}' has no 'config_var' property") + cfg_var: Any = qstn["config_var"] if not isinstance(cfg_var, str): - raise ValueError("question in '{file}' has malformed 'config_var' property ({cfg_var})".format(file=file, cfg_var=cfg_var)) + raise ValueError(f"question in '{file}' has malformed 'config_var' property ({cfg_var})") del qstn["config_var"] if qstn: @@ -713,7 +723,7 @@ def unmarshal_config(dct): # type: (dict) -> dict[str, list[Question]] return ret -def write_encryption_key(aes_key_location): # type: (str) -> None +def write_encryption_key(aes_key_location: str): """ Creates an AES encryption key for the postgres traffic vault backend @@ -728,11 +738,11 @@ def write_encryption_key(aes_key_location): # type: (str) -> None "-base64", "32" ) - if not exec_openssl("Generating an AES encryption key to {loc}".format(loc=aes_key_location), *args): + if not exec_openssl(f"Generating an AES encryption key to {aes_key_location}", *args): logging.debug("AES key generation failed") raise OSError("failed to generate AES key") -def exec_openssl(description, *cmd_args): # type: (str, ...) -> bool +def exec_openssl(description: str, *cmd_args: str) -> bool: """ Executes openssl with the supplied command-line arguments. @@ -757,13 +767,13 @@ def exec_openssl(description, *cmd_args): # type: (str, ...) -> bool output = proc.communicate() logging.debug("openssl exec failed with code %s; stderr: %s", proc.returncode, output[1]) while True: - ans = input("{description} failed. Try again (y/n) [y]: ".format(description=description)) - if not ans or ans.lower().startswith('n'): + ans = input(f"{description} failed. Try again (y/n) [y]: ") + if not ans or ans.lower().startswith("n"): return False - if ans.lower().startswith('y'): + if ans.lower().startswith("y"): break -def setup_certificates(conf, root, ops_user, ops_group): # type: (SSLConfig, str, str, str) -> int +def setup_certificates(conf: SSLConfig, root: str, ops_user: str, ops_group: str) -> int: """ Generates self-signed SSL certificates from the given configuration. :returns: For whatever reason this subroutine needs to dictate the return code of the script, so that's what it returns. @@ -791,7 +801,7 @@ def setup_certificates(conf, root, ops_user, ops_group): # type: (SSLConfig, str "-out", "server.key", "-passout", - "pass:{rsa_password}".format(rsa_password=conf.rsa_password), + f"pass:{conf.rsa_password}", "1024" ) if not exec_openssl("Generating an RSA Private Server Key", *args): @@ -805,7 +815,7 @@ def setup_certificates(conf, root, ops_user, ops_group): # type: (SSLConfig, str "-out", "server.csr", "-passin", - "pass:{rsa_password}".format(rsa_password=conf.rsa_password), + f"pass:{conf.rsa_password}", "-subj", conf.params ) @@ -822,7 +832,7 @@ def setup_certificates(conf, root, ops_user, ops_group): # type: (SSLConfig, str "-out", "server.key", "-passin", - "pass:{rsa_password}".format(rsa_password=conf.rsa_password) + f"pass:{conf.rsa_password}" ) if not exec_openssl("Removing the pass phrase from the server key", *args): return 1 @@ -879,12 +889,10 @@ def setup_certificates(conf, root, ops_user, ops_group): # type: (SSLConfig, str cdn_conf_path = os.path.join(root, "opt/traffic_ops/app/conf/cdn.conf") try: - with open(cdn_conf_path) as conf_file: + with open(cdn_conf_path, encoding="utf-8") as conf_file: cdn_conf = json.load(conf_file) except (OSError, ValueError) as e: - exception = OSError("reading {cdn_conf_path}: {e}".format(cdn_conf_path=cdn_conf_path, e=e)) - exception.__cause__ = e - raise exception + raise OSError(f"reading {cdn_conf_path}: {e}") from e if ( not isinstance(cdn_conf, dict) or @@ -894,7 +902,7 @@ def setup_certificates(conf, root, ops_user, ops_group): # type: (SSLConfig, str logging.critical("Malformed %s; improper object and/or missing 'traffic_ops_golang' key", cdn_conf_path) return 1 - to_golang = cdn_conf["traffic_ops_golang"] + to_golang: Any = cdn_conf["traffic_ops_golang"] if ( "cert" not in to_golang or not isinstance(to_golang["cert"], str) @@ -915,15 +923,15 @@ def setup_certificates(conf, root, ops_user, ops_group): # type: (SSLConfig, str return 0 -def random_word(length = 12): # type: (int) -> str +def random_word(length: int = 12) -> str: """ Returns a randomly generated string 'length' characters long containing only word characters ([a-zA-Z0-9_]). """ - word_chars = string.ascii_letters + string.digits + '_' - return ''.join(random.choice(word_chars) for _ in range(length)) + word_chars = string.ascii_letters + string.digits + "_" + return "".join(random.choice(word_chars) for _ in range(length)) -def generate_cdn_conf(questions, fname, automatic, root): # type: (list[Question], str, bool, str) -> bool +def generate_cdn_conf(questions: List[Question], fname: str, automatic: bool, root: str) -> bool: """ Generates some properties of a cdn.conf file based on the passed questions. @@ -940,62 +948,45 @@ def generate_cdn_conf(questions, fname, automatic, root): # type: (list[Question try: num_secrets = int(cdn_conf["keepSecrets"]) except KeyError as e: - exception = ValueError("missing 'keepSecrets' config_var") - exception.__cause__ = e - raise exception + raise ValueError("missing 'keepSecrets' config_var") from e except ValueError as e: - exception = ValueError("invalid 'keepSecrets' config_var value: {e}".format(e=e)) - exception.__cause__ = e - raise exception + raise ValueError(f"invalid 'keepSecrets' config_var value: {e}") from e try: - port = cdn_conf["port"] # type: str + port = cdn_conf["port"] + int(port) except KeyError as e: - exception = ValueError("missing 'port' config_var") - exception.__cause__ = e - raise exception + raise ValueError("missing 'port' config_var") from e except ValueError as e: - exception = ValueError("invalid 'port' config_var value: {e}".format(e=e)) - exception.__cause__ = e - raise exception + raise ValueError(f"invalid 'port' config_var value: {e}") from e try: workers = int(cdn_conf["workers"]) except KeyError as e: - exception = ValueError("missing 'workers' config_var") - exception.__cause__ = e - raise exception + raise ValueError("missing 'workers' config_var") from e except ValueError as e: - exception = ValueError("invalid 'workers' config_var value: {e}".format(e=e)) - exception.__cause__ = e - raise exception + raise ValueError(f"invalid 'workers' config_var value: {e}") from e try: url = cdn_conf["base_url"] except KeyError as e: - exception = ValueError("missing 'base_url' config_var") - exception.__cause__ = e - raise exception + raise ValueError("missing 'base_url' config_var") from e try: ldap_loc = cdn_conf["ldap_conf_location"] except KeyError as e: - exception = ValueError("missing 'ldap_conf_location' config_var") - exception.__cause__ = e - raise exception + raise ValueError("missing 'ldap_conf_location' config_var") from e conf = CDNConfig(gen_secret, num_secrets, port, workers, url, ldap_loc) path = os.path.join(root, fname.lstrip('/')) - existing_conf = {} + existing_conf: Dict[Any, Any] = {} if os.path.isfile(path): - with open(path) as conf_file: + with open(path, encoding="utf-8") as conf_file: try: existing_conf = json.load(conf_file) except ValueError as e: - exception = ValueError("invalid existing cdn.config at {path}: {e}".format(path=path, e=e)) - exception.__cause__ = e - raise exception + raise ValueError(f"invalid existing cdn.config at {path}: {e}") from e if not isinstance(existing_conf, dict): logging.warning("Existing cdn.conf (at '%s') is not an object - overwriting", path) @@ -1018,25 +1009,25 @@ def generate_cdn_conf(questions, fname, automatic, root): # type: (list[Question traffic_vault_backend = "postgres" tv_aes_key_location = os.path.join(root, TRAFFIC_VAULT_AES_KEY_FILE.lstrip('/')) - with open(path, "w+") as conf_file: - json.dump(existing_conf, conf_file, indent=indent) + with open(path, "w+", encoding="utf-8") as conf_file: + json.dump(existing_conf, conf_file, indent=INDENT) print(file=conf_file) logging.info("CDN configuration has been saved") try: - traffic_vault_backend = existing_conf["traffic_ops_golang"]["traffic_vault_backend"] + traffic_vault_backend: Any = existing_conf["traffic_ops_golang"]["traffic_vault_backend"] except KeyError as e: logging.warning("no traffic vault backend configured, using default postgres") if traffic_vault_backend == "postgres": try: - tv_aes_key_location = existing_conf["traffic_ops_golang"]["traffic_vault_config"]["aes_key_location"] + tv_aes_key_location: Any = existing_conf["traffic_ops_golang"]["traffic_vault_config"]["aes_key_location"] except KeyError as e: logging.warning("no traffic vault aes encryption key location specified, using default %s", TRAFFIC_VAULT_AES_KEY_FILE) write_encryption_key(tv_aes_key_location) return traffic_vault_backend == "postgres" -def db_connection_string(dbconf): # type: (dict) -> str +def db_connection_string(dbconf: Dict[str, Any]) -> str: """ Constructs a database connection string from the passed configuration object. """ @@ -1045,9 +1036,9 @@ def db_connection_string(dbconf): # type: (dict) -> str db_name = "traffic_ops" if dbconf["type"] == "Pg" else dbconf["type"] hostname = dbconf["hostname"] port = dbconf["port"] - return "postgresql://{user}:{password}@{hostname}:{port}/{db_name}".format(user=user, password=password, hostname=hostname, port=port, db_name=db_name) + return f"postgresql://{user}:{password}@{hostname}:{port}/{db_name}" -def exec_psql(conn_str, query, **args): # type: (str, str, dict) -> str +def exec_psql(conn_str: str, query: str, **args: str) -> str: """ Executes SQL queries by forking and exec-ing '/usr/bin/psql'. @@ -1068,14 +1059,11 @@ def exec_psql(conn_str, query, **args): # type: (str, str, dict) -> str if proc.returncode != 0: logging.debug("psql exec failed; stderr: %s\n\tstdout: %s", output[1], output[0]) raise OSError("failed to execute database query") - if sys.version_info.major >= 3: - return output[0].strip() - else: - return string.strip(output[0]) + return output[0].strip() -def invoke_db_admin_pl(action, root, tv): # type: (str, str, bool) -> None +def invoke_db_admin_pl(action: str, root: str, tv: bool): """ - Exectues admin with the given action, and looks for it from the given root directory. + Executes admin with the given action, and looks for it from the given root directory. """ path = os.path.join(root, "opt/traffic_ops/app") # This is a workaround for admin using hard-coded relative paths. That @@ -1090,13 +1078,19 @@ def invoke_db_admin_pl(action, root, tv): # type: (str, str, bool) -> None stdout=subprocess.PIPE, universal_newlines=True, ) - output = proc.communicate() # type: str + output = proc.communicate() if proc.returncode != 0: logging.debug("admin exec failed; stderr: %s\n\tstdout: %s", output[1], output[0]) - raise OSError("Database {action} failed".format(action=action)) + raise OSError(f"Database {action} failed") logging.info("Database %s succeeded", action) -def setup_database_data(conn_str, user, param_conf, root, postgresTV): # type: (str, User, dict, str, bool) -> None +def setup_database_data( + conn_str: str, + user: User, + param_conf: Dict[str, str], + root: str, + postgresTV: bool +): """ Sets up all necessary initial database data using `/usr/bin/sql` """ @@ -1126,32 +1120,32 @@ def setup_database_data(conn_str, user, param_conf, root, postgresTV): # type: ( invoke_db_admin_pl("migrate", root, True) hashed_pass = hash_pass(user.password) - insert_admin_query = ''' + insert_admin_query = f''' INSERT INTO tm_user (username, tenant_id, role, local_passwd, confirm_local_passwd) VALUES ( - '{}', + '{user.username}', (SELECT id FROM tenant WHERE name = 'root'), (SELECT id FROM role WHERE name = 'admin'), '{hashed_pass}', '{hashed_pass}' ) ON CONFLICT (username) DO NOTHING; - '''.format(user.username, hashed_pass=hashed_pass) + ''' _ = exec_psql(conn_str, insert_admin_query) logging.info("=========== Setting up cdn") - insert_cdn_query = "\n\t-- global parameters" + ''' + insert_cdn_query = "\n\t-- global parameters" + f''' INSERT INTO cdn (name, domain_name, dnssec_enabled) - VALUES ('{cdn_name}', '{dns_subdomain}', false) + VALUES ('{param_conf["cdn_name"]}', '{param_conf["dns_subdomain"]}', false) ON CONFLICT DO NOTHING; - '''.format(**param_conf) + ''' logging.info("\n%s", insert_cdn_query) _ = exec_psql(conn_str, insert_cdn_query) tm_url = param_conf["tm.url"] logging.info("=========== Setting up parameters") - insert_parameters_query = "\n\t-- global parameters" + ''' + insert_parameters_query = "\n\t-- global parameters" + f''' INSERT INTO parameter (name, config_file, value) VALUES ('tm.url', 'global', '{tm_url}'), ('tm.infourl', 'global', '{tm_url}/doc'), @@ -1159,12 +1153,12 @@ def setup_database_data(conn_str, user, param_conf, root, postgresTV): # type: ( ('geolocation.polling.url', 'CRConfig.json', '{tm_url}/routing/GeoLite2-City.mmdb.gz'), ('geolocation6.polling.url', 'CRConfig.json', '{tm_url}/routing/GeoLiteCityv6.dat.jz') ON CONFLICT (name, config_file, value) DO NOTHING; - '''.format(tm_url=tm_url) + ''' logging.info("\n%s", insert_parameters_query) _ = exec_psql(conn_str, insert_parameters_query) logging.info("\n=========== Setting up profiles") - insert_profiles_query = "\n\t-- global parameters" + ''' + insert_profiles_query = "\n\t-- global parameters" + f''' INSERT INTO profile (name, description, type, cdn) VALUES ('GLOBAL' 'Global Traffic Ops profile, DO NOT DELETE', 'UNK_PROFILE', (SELECT id FROM cdn WHERE name='ALL')) ON CONFLICT DO NOTHING; @@ -1212,26 +1206,25 @@ def setup_database_data(conn_str, user, param_conf, root, postgresTV): # type: ( ) ) ON CONFLICT (profile, parameter) DO NOTHING; - '''.format(tm_url=tm_url) + ''' logging.info("\n%s", insert_profiles_query) _ = exec_psql(conn_str, insert_cdn_query) def main( -automatic, # type: bool -debug, # type: bool -defaults, # type: str -cfile, # type: str -root_dir, # type: str -ops_user, # type: str -ops_group, # type: str -no_restart_to, # type: bool -no_database, # type: bool -): + automatic: bool, + debug: bool, + defaults: Optional[str], + cfile: str, + root_dir: str, + ops_user: str, + ops_group: str, + no_restart_to: bool, + no_database: bool, +) -> int: """ Runs the main routine given the parsed arguments as input. - :rtype: int """ - postgresTV = False + postgres_tv = False if debug: logging.getLogger().setLevel(logging.DEBUG) else: @@ -1251,13 +1244,13 @@ def main( try: if defaults: try: - with open(defaults, "w") as dump_file: - json.dump(DEFAULTS, dump_file, indent=indent) + with open(defaults, "w", encoding="utf-8") as dump_file: + json.dump(DEFAULTS, dump_file, indent=INDENT) except OSError as e: logging.critical("Writing output: %s", e) return 1 else: - json.dump(DEFAULTS, sys.stdout, cls=ConfigEncoder, indent=indent) + json.dump(DEFAULTS, sys.stdout, cls=ConfigEncoder, indent=INDENT) print() except ValueError as e: logging.critical("Converting defaults to JSON: %s", e) @@ -1270,7 +1263,7 @@ def main( else: logging.info("Using input file %s", cfile) try: - with open(cfile) as conf_file: + with open(cfile, encoding="utf-8") as conf_file: user_input = unmarshal_config(json.load(conf_file)) diffs = sanity_check_config(user_input, automatic) logging.info( @@ -1291,17 +1284,17 @@ def main( generate_todb_conf(TV_DB_CONF_FILE, root_dir, tv_dbconf) generate_ldap_conf(user_input[LDAP_CONF_FILE], LDAP_CONF_FILE, automatic, root_dir) admin_conf = generate_users_conf( - user_input[USERS_CONF_FILE], - USERS_CONF_FILE, - automatic, - root_dir + user_input[USERS_CONF_FILE], + USERS_CONF_FILE, + automatic, + root_dir ) generate_profiles_dir(user_input[PROFILES_CONF_FILE]) opensslconf = generate_openssl_conf(user_input[OPENSSL_CONF_FILE], OPENSSL_CONF_FILE, automatic) paramconf = generate_param_conf(user_input[PARAM_CONF_FILE], PARAM_CONF_FILE, automatic, root_dir) postinstall_cfg = os.path.join(root_dir, POST_INSTALL_CFG.lstrip('/')) if not os.path.isfile(postinstall_cfg): - with open(postinstall_cfg, 'w+') as conf_file: + with open(postinstall_cfg, "w+", encoding="utf-8") as conf_file: print("{}", file=conf_file) except OSError as e: logging.critical("Writing configuration: %s", e) @@ -1319,7 +1312,7 @@ def main( return 1 try: - postgresTV = generate_cdn_conf(user_input[CDN_CONF_FILE], CDN_CONF_FILE, automatic, root_dir) + postgres_tv = generate_cdn_conf(user_input[CDN_CONF_FILE], CDN_CONF_FILE, automatic, root_dir) except OSError as e: logging.critical("Generating cdn.conf: %s", e) return 1 @@ -1347,7 +1340,7 @@ def db_connect_failed(): ) try: - setup_database_data(conn_str, admin_conf, paramconf, root_dir, postgresTV) + setup_database_data(conn_str, admin_conf, paramconf, root_dir, postgres_tv) except (subprocess.CalledProcessError, OSError) as e: db_connect_failed() return 1 @@ -1360,19 +1353,20 @@ def db_connect_failed(): logging.info("Starting Traffic Ops") try: cmd = ["/sbin/service", "traffic_ops", "restart"] - proc = subprocess.Popen( + with subprocess.Popen( cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True, - ) - if proc.wait(): - raise subprocess.CalledProcessError(proc.returncode, cmd) - except (subprocess.CalledProcessError, OSError) as e: - output = proc.communicate() - logging.critical("Failed to restart Traffic Ops, return code %s: %s", e.returncode, e) - logging.debug("stderr: %s\n\tstdout: %s", output[1], output[0]) - return 1 + ) as proc: + try: + if proc.wait(): + raise subprocess.CalledProcessError(proc.returncode, cmd) + except (subprocess.CalledProcessError) as e: + output = proc.communicate() + logging.critical("Failed to restart Traffic Ops, return code %s: %s", e.returncode, e) + logging.debug("stderr: %s\n\tstdout: %s", output[1], output[0]) + return 1 except OSError as e: logging.critical("Failed to restart Traffic Ops: unknown error occurred: %s", e) return 1 @@ -1508,15 +1502,15 @@ def db_connect_failed(): try: EXIT_CODE = main( - ARGS.automatic, - DEBUG, - DEFAULTS_ARG, - CFILE, - os.path.abspath(ARGS.root_directory), - ARGS.ops_user, - ARGS.ops_group, - ARGS.no_restart_to, - ARGS.no_database + ARGS.automatic, + DEBUG, + DEFAULTS_ARG, + CFILE, + os.path.abspath(ARGS.root_directory), + ARGS.ops_user, + ARGS.ops_group, + ARGS.no_restart_to, + ARGS.no_database ) sys.exit(EXIT_CODE) except KeyboardInterrupt: diff --git a/traffic_ops/install/bin/input.json b/traffic_ops/install/bin/input.json index ad1b601e79..1dc6e4bf3d 100644 --- a/traffic_ops/install/bin/input.json +++ b/traffic_ops/install/bin/input.json @@ -33,7 +33,7 @@ { "LDAP Admin Password": "", "config_var": "password", - "hidden": "1" + "hidden": true }, { "LDAP Search Base": "", @@ -64,7 +64,7 @@ { "Traffic Ops database password": "dbpass", "config_var": "password", - "hidden": "1" + "hidden": true } ], "/opt/traffic_ops/app/conf/production/tv.conf": [ @@ -91,7 +91,7 @@ { "Traffic Ops database password": "dbpass", "config_var": "password", - "hidden": "1" + "hidden": true } ], "/opt/traffic_ops/install/data/json/openssl_configuration.json": [ @@ -126,7 +126,7 @@ { "RSA Passphrase": "password", "config_var": "rsaPassword", - "hidden": "1" + "hidden": true } ], "/opt/traffic_ops/install/data/json/profiles.json": [ @@ -151,7 +151,7 @@ { "Password for the admin user": "twelve", "config_var": "tmAdminPw", - "hidden": "1" + "hidden": true } ], "/opt/traffic_ops/install/data/profiles/": [ diff --git a/traffic_ops/install/bin/postinstall b/traffic_ops/install/bin/postinstall index 74f46d01bd..97bab4fc70 100755 --- a/traffic_ops/install/bin/postinstall +++ b/traffic_ops/install/bin/postinstall @@ -26,9 +26,9 @@ for arg in "$@"; do done PATH+=:/usr/libexec/ -python_bin="$(command -v {python,platform-python}{3{{.,}{9,8,7,6},},2{7,.7,},} | head -n1)" +python_bin="$(command -v {python,platform-python}{3{{.,}{12..6},},} | head -n1)" if [[ -z "$python_bin" ]]; then - echo 'No python3 or python2 executable was found. Python is required to run the Postinstall script.' >/dev/stderr + echo 'No python3 executable was found. Python is required to run the Postinstall script.' >/dev/stderr exit 1 fi diff --git a/traffic_ops/install/bin/postinstall.test.sh b/traffic_ops/install/bin/postinstall.test.sh index 5507e3c58d..0f37db4cdb 100755 --- a/traffic_ops/install/bin/postinstall.test.sh +++ b/traffic_ops/install/bin/postinstall.test.sh @@ -22,34 +22,25 @@ readonly MY_DIR="$(pwd)"; help_string="$(<<-'HELP_STRING' cat Usage: ./postinstall.test.h [ - -2 Set Python version to 2 - -3 Set Python version to 3 -b Explicitly set the path to the Python binary as this value -h, ? Print this help text and exit - -s Do not test Python 2 after testing Python 3 HELP_STRING )" -while getopts :23hsb: opt; do +while getopts :hb: opt; do case "$opt" in - 2) python_version=2;; - 3) python_version=3;; b) python_bin="$OPTARG";; h) echo "$help_string" && exit;; - s) skip_python2=true;; ?) echo "$help_string" && exit;; *) echo "Invalid flag received: ${OPTARG}" >&2 && echo "$help_string" && exit 1;; esac; done; -python_version="${python_version:-3}"; -python_bin="${python_bin:-/usr/bin/python${python_version}}"; +python_bin="${python_bin:-/usr/bin/python3}"; -if [[ ! -x "$python_bin" && "$python_version" -ge 3 ]]; then +if [[ ! -x "$python_bin" ]]; then echo "Python 3.6+ is required to run - or test - _postinstall.py" >&2; exit 1; -elif [[ ! -x "$python_bin" && "$python_version" == 2 ]]; then - echo "Python ${python_version} is required to run - or test - _postinstall.py against Python 2" >&2; fi readonly TO_PASSWORD=twelve; @@ -58,7 +49,6 @@ readonly ROOT_DIR="$(mktemp -d)"; trap 'rm -rf $ROOT_DIR' EXIT; "$python_bin" < "$ROOT_DIR/opt/traffic_ops/app/conf/cdn.conf" </dev/null | tee -a "${ROOT_DIR}/stdout"; -from __future__ import print_function import subprocess import sys import _postinstall @@ -359,13 +348,9 @@ fi readonly USERS_JSON_FILE="$ROOT_DIR/opt/traffic_ops/install/data/json/users.json"; "$python_bin" <&2; exit 1; fi - -if [[ "$python_version" != 2 && -z "$skip_python2" ]]; then - exec "${MY_DIR}/$(basename "${BASH_SOURCE[0]}")" -2; -fi;