diff --git a/.eslintrc.json b/.eslintrc.json index 161db2e26..b2e86e9ee 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -34,13 +34,14 @@ "requireReturn": false }], "no-unused-vars": ["warn"], - "operator-linebreak": ["error", "after", { "overrides": { "?": "ignore", ":": "ignore" } }] + "operator-linebreak": ["error", "after", { "overrides": { "?": "ignore", ":": "ignore" } }], + "no-multiple-empty-lines": ["error"] }, "ignorePatterns": [ "vendor/*", "config/*", "resources/docker", - "tests/fixtures" + "tests/fixtures/samples/sample-docker*.yml" ], "overrides": [ { diff --git a/CHANGELOG.md b/CHANGELOG.md index dee959d11..f8a095036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Toolkit change log +## Version 10.29.1 + - DQA-0000: Add xdebug_break() to the list of forbidden functions. + - DQA-14193: Gitleaks improvements + +## Version 10.28.0 + - DQA-13950: Remove dependency on ec-europa/qa-automation and add support to coder 9. + - DQA-13914: Validate SHA256 of gitleaks binary after download. + - DQA-13914: Verify SHA1 after dump is downloaded and retry in case of issues during download. + - DQA-14171: Improvements run-tests commands. + +## Version 10.27.0 + - DQA-13733: Set new gitlab & nextcloud urls. + - DQA-13747: Support Coder 9.0. + - DQA-13741: PHP 8.5 compatibility. + - DQA-13613: Check composer-patches v2. + - DQA-13523: Adapt DQA for new gitlab instance. + - DQA-13626: Improve database default values. + - DQA-13253: Prevent usage of eslint >=9, to align with drupal/core. + ## Version 10.26.2 - DROPSEC-16324: Fix opts.yml loader default. - DQA-13173: Added dependency required by Drupal 11.3.x. diff --git a/composer.json b/composer.json index ba5f17d07..f972184d9 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,6 @@ "dantleech/gherkin-lint": "^0.2.3", "drupal/coder": "^8.3.10", "drush/drush": "^11.0.4 || ^12.0 || ^13.0", - "ec-europa/qa-automation": "^9.4.0", "ec-europa/toolkit-composer-plugin": "^0.0.1", "guzzlehttp/guzzle": "^6.3 || ^7.0", "jakeasmith/http_build_url": "^1.0", @@ -56,11 +55,13 @@ "moodlehq/moodle-plugin-ci": "^4.5", "pear/archive_tar": "^1.4", "php-parallel-lint/php-parallel-lint": "^1.3", + "phpcompatibility/php-compatibility": "^9.3 || ^10.0@alpha", "phpmd/phpmd": "^2.12", "phpstan/phpstan": "^1.12.15 || ^2.1", "phpstan/phpstan-deprecation-rules": "^1.0 || ^2.0", "phpunit/phpunit": "^9.5 || ^10.0 || ^11.0", - "squizlabs/php_codesniffer": "^3.7", + "squizlabs/php_codesniffer": "^3.7 || ^4.0", + "symfony/yaml": "^5.0 || ^6.0 || ^7.0", "vlucas/phpdotenv": "^5.6" }, "suggest": { @@ -89,13 +90,13 @@ } }, "scripts": { - "toolkit-update-phpcs-config": "$(pwd)/vendor/bin/phpcs --config-set installed_paths '../../drupal/coder/coder_sniffer,../../phpcompatibility/php-compatibility,../../ec-europa/qa-automation/phpcs'", + "toolkit-update-phpcs-config": "$(pwd)/vendor/bin/phpcs --config-set installed_paths '../../drupal/coder/coder_sniffer,../../phpcompatibility/php-compatibility'", "post-install-cmd": [ - "./vendor/bin/phpcs --config-set installed_paths '../../drupal/coder/coder_sniffer,../../phpcompatibility/php-compatibility,phpcs,../../ec-europa/qa-automation/phpcs'", + "./vendor/bin/phpcs --config-set installed_paths '../../drupal/coder/coder_sniffer,../../phpcompatibility/php-compatibility,phpcs'", "php -r \"file_exists('vendor/bin/run') || (copy('run', 'vendor/bin/run') && chmod('vendor/bin/run', fileperms('run')));\"" ], "post-update-cmd": [ - "./vendor/bin/phpcs --config-set installed_paths '../../drupal/coder/coder_sniffer,../../phpcompatibility/php-compatibility,phpcs,../../ec-europa/qa-automation/phpcs'", + "./vendor/bin/phpcs --config-set installed_paths '../../drupal/coder/coder_sniffer,../../phpcompatibility/php-compatibility,phpcs'", "php -r \"file_exists('vendor/bin/run') || (copy('run', 'vendor/bin/run') && chmod('vendor/bin/run', fileperms('run')));\"" ] } diff --git a/config/commands/code-review.yml b/config/commands/code-review.yml index 6eea00f4f..0813467c3 100644 --- a/config/commands/code-review.yml +++ b/config/commands/code-review.yml @@ -16,7 +16,4 @@ command: requirements: false run-tests: options: - phpunit: false - behat: false - blackfire: false - components: false + type: ${toolkit.run-tests.type} diff --git a/config/commands/gitleaks.yml b/config/commands/gitleaks.yml index e321435e6..df0d805cf 100644 --- a/config/commands/gitleaks.yml +++ b/config/commands/gitleaks.yml @@ -3,5 +3,8 @@ command: run-gitleaks: options: tag: ${gitleaks.tag} + sha256: ${gitleaks.sha256} os: ${gitleaks.os} options: ${gitleaks.options} + ignore-file: ${gitleaks.ignore-file} + report-to-file: ${gitleaks.report-to-file} diff --git a/config/runner/gitleaks.yml b/config/runner/gitleaks.yml index d1938c386..0ca51e2d7 100644 --- a/config/runner/gitleaks.yml +++ b/config/runner/gitleaks.yml @@ -1,5 +1,12 @@ gitleaks: repo: https://github.com/gitleaks/gitleaks tag: 8.27.2 + sha256: 141c3b2dede46d8b3a53b47116da756bd223decc0374797559a6b50ecba5590c os: linux_x64 options: '--no-banner -v --no-git --follow-symlinks' + ignore-file: '.leaksignore' + report-to-file: false + fields: + - File + - Fingerprint + - ToolkitFingerprint diff --git a/config/runner/toolkit-test.yml b/config/runner/toolkit-test.yml index fc1da22cf..d9f392a55 100644 --- a/config/runner/toolkit-test.yml +++ b/config/runner/toolkit-test.yml @@ -35,7 +35,7 @@ toolkit: standards: - ./vendor/drupal/coder/coder_sniffer/Drupal - ./vendor/drupal/coder/coder_sniffer/DrupalPractice - - ./vendor/ec-europa/qa-automation/phpcs/QualityAssurance + - ./vendor/ec-europa/toolkit/phpcs/QualityAssurance ignore_patterns: - ${toolkit.build.dist.root}/ - .cache/ @@ -126,3 +126,14 @@ toolkit: allow-list: axe-scan-allowlist.csv run-summary: true summary-result-file: axe-scan-summary.csv + run-tests: + type: clone + phpunit: + clone: false + clean: false + behat: + clone: true + clean: true + blackfire: + clone: false + clean: false diff --git a/config/runner/toolkit.yml b/config/runner/toolkit.yml index 5c6dda6b0..6c9eedbba 100644 --- a/config/runner/toolkit.yml +++ b/config/runner/toolkit.yml @@ -9,10 +9,11 @@ toolkit: mydumper: '/scripts/backupdb.sh' type: 'nextcloud' show_progress: false + download_retries: 3 nextcloud: # Nextcloud supports the following services: mysql, virtuoso, solr. services: 'mysql' - url: 'files.fpfis.tech.ec.europa.eu/remote.php/dav/files' + url: 'files.dhs.tech.ec.europa.eu/remote.php/dav/files' admin: false vendor: 'ec-europa' source: 'reference' diff --git a/docker-compose.yml b/docker-compose.yml index 5f6177cdb..75292533b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: web: - image: registry.fpfis.eu/digit/devsecops/httpd-php:8.3-ci + image: registry.dhs.tech.ec.europa.eu/digit/devsecops/httpd-php:8.3-ci working_dir: /__w/toolkit/toolkit ports: # Use port "8080:8080" to access the mock in the browser OR to use xdebug. diff --git a/docs/guide/testing-project.rst b/docs/guide/testing-project.rst index 7c02102c7..9758ff95a 100755 --- a/docs/guide/testing-project.rst +++ b/docs/guide/testing-project.rst @@ -109,7 +109,7 @@ This will enforce the usage of the following standards: vendor/drupal/coder/coder_sniffer/Drupal vendor/drupal/coder/coder_sniffer/DrupalPractice - vendor/ec-europa/qa-automation/phpcs/QualityAssurance + vendor/ec-europa/toolkit/phpcs/QualityAssurance These are the default configurations that can be overridden in the `runner.yml.dist and/or runner.yml `_ file. @@ -126,7 +126,7 @@ These are the default configurations that can be overridden in the `runner.yml.d standards: - ./vendor/drupal/coder/coder_sniffer/Drupal - ./vendor/drupal/coder/coder_sniffer/DrupalPractice - - ./vendor/ec-europa/qa-automation/phpcs/QualityAssurance + - ./vendor/ec-europa/toolkit/phpcs/QualityAssurance ignore_patterns: - vendor/ - web/ diff --git a/phpcs.xml b/phpcs.xml index 5e31e88f5..45834dd11 100644 --- a/phpcs.xml +++ b/phpcs.xml @@ -23,8 +23,6 @@ tests/fixtures/ tests/sandbox/ - - @@ -37,12 +35,14 @@ + + @@ -241,26 +241,6 @@ 0 - - - 0 - - - - 0 - - - - - - - - - - - - - @@ -300,7 +280,6 @@ - @@ -312,9 +291,6 @@ - - - @@ -332,9 +308,7 @@ - - diff --git a/phpcs/QualityAssurance/Sniffs/Functions/DrupalDeprecatedSniff.php b/phpcs/QualityAssurance/Sniffs/Functions/DrupalDeprecatedSniff.php new file mode 100644 index 000000000..a2b19c722 --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/Functions/DrupalDeprecatedSniff.php @@ -0,0 +1,76 @@ + 'dump']; + + /** + * If true, an error will be thrown; otherwise a warning. + * + * @var boolean + */ + public $error = true; + + + /** + * Generates the error or warning for this sniff. + * + * @param \PHP_CodeSniffer\Files\File $phpcsFile The file being scanned. + * @param int $stackPtr The position of the forbidden function + * in the token array. + * @param string $function The name of the forbidden function. + * @param string $pattern The pattern used for the match. + * + * @return void + */ + protected function addError($phpcsFile, $stackPtr, $function, $pattern=null) + { + $data = [$function]; + $error = 'Function %s() has been deprecated'; + $type = 'Deprecated'; + + if (empty($this->forbiddenFunctions[$function]) === false) { + $data[] = $this->forbiddenFunctions[$function]; + $error .= '; use %s() instead'; + } + + if ($this->error === true) { + $phpcsFile->addError($error, $stackPtr, $type, $data); + } else { + $phpcsFile->addWarning($error, $stackPtr, $type, $data); + } + + }//end addError() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/Functions/DrupalForbiddenHooksSniff.php b/phpcs/QualityAssurance/Sniffs/Functions/DrupalForbiddenHooksSniff.php new file mode 100644 index 000000000..a95456941 --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/Functions/DrupalForbiddenHooksSniff.php @@ -0,0 +1,100 @@ + 'hook_form_FORM_ID_alter() or hook_form_BASE_FORM_ID_alter()']; + + /** + * A list of file extensions to check. + * + * @var array + */ + public $extensions = [ + 'inc', + 'module', + 'theme', + ]; + + + /** + * Returns an array of tokens this test wants to listen for. + * + * @return array + */ + public function register() + { + return [T_FUNCTION]; + + }//end register() + + + /** + * Processes this test, when one of its tokens is encountered. + * + * @param File $phpcsFile The file being scanned. + * @param int $stackPtr The position of the current token + * in the stack passed in $tokens. + * + * @return void + */ + public function process(File $phpcsFile, $stackPtr) + { + $filename = basename($phpcsFile->getFilename()); + $exploded = explode('.', $filename); + $extension = end($exploded); + if (false === in_array($extension, $this->extensions)) { + return; + } + + $module = $exploded[0]; + $tokens = $phpcsFile->getTokens(); + $function = $tokens[($stackPtr + 2)]['content']; + foreach ($this->forbiddenHooks as $hook => $replacement) { + if (true === ($function === str_replace('hook', $module, $hook))) { + $warning = 'The usage of the hook %s() is forbidden'; + $data = [$hook]; + if (false === empty($replacement)) { + $warning .= ', instead use %s.'; + $data[] = $replacement; + } else { + $warning .= '.'; + } + + $phpcsFile->addError($warning, $stackPtr, 'ForbiddenHook', $data); + } + } + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/Functions/DrupalHttpRequestSniff.php b/phpcs/QualityAssurance/Sniffs/Functions/DrupalHttpRequestSniff.php new file mode 100644 index 000000000..5684f9a38 --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/Functions/DrupalHttpRequestSniff.php @@ -0,0 +1,44 @@ + '\Drupal::httpClient']; + + /** + * If true, an error will be thrown; otherwise a warning. + * + * @var boolean + */ + public $error = true; + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/Functions/DrupalWrappersSniff.php b/phpcs/QualityAssurance/Sniffs/Functions/DrupalWrappersSniff.php new file mode 100644 index 000000000..a0a863eeb --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/Functions/DrupalWrappersSniff.php @@ -0,0 +1,67 @@ + 'drupal_register_shutdown_function', + 'set_time_limit' => 'drupal_set_time_limit', + 'xml_parser_create' => 'drupal_xml_parser_create', + 'chmod' => 'FileSystemInterface::chmod', + 'dirname' => 'FileSystemInterface::dirname', + 'mkdir' => 'FileSystemInterface::mkdir', + 'move_uploaded_file' => 'FileSystemInterface::moveUploadedFile', + 'rmdir' => 'FileSystemInterface::rmdir', + 'tempnam' => 'FileSystemInterface::tempnam', + 'unlink' => 'FileSystemInterface::unlink', + 'lcfirst' => 'Unicode::lcfirst', + 'ucwords' => 'Unicode::ucwords', + 'http_build_query' => 'UrlHelper::buildQuery', + 'parse_url' => 'UrlHelper::parse', + + 'ucfirst' => 'Unicode::ucfirst', + 'copy' => 'FileSystemInterface::copy', + 'rename' => 'FileSystemInterface::move', + 'substr' => 'mb_substr', + 'strtolower' => 'mb_strtolower', + 'strtoupper' => 'mb_strtoupper', + 'date' => 'Drupal::service("date.formatter")->format', + 'xdebug_break' => null, + ]; + + /** + * If true, an error will be thrown; otherwise a warning. + * + * @var boolean + */ + public $error = true; + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/Generic/CredentialsSniff.php b/phpcs/QualityAssurance/Sniffs/Generic/CredentialsSniff.php new file mode 100644 index 000000000..3244fe6f3 --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/Generic/CredentialsSniff.php @@ -0,0 +1,101 @@ +getTokens()) + 1); + $filePath = $phpcsFile->getFilename(); + $fileName = basename($filePath); + + if (preg_match('/^docker-compose*/i', $fileName) !== 1) { + return $end; + } + + $fileContent = file($filePath); + $checkEnvVars = [ + 'asda_user', + 'asda_password', + 'nextcloud_user', + 'nextcloud_pass', + 'api_token', + ]; + try { + $yaml = Yaml::parseFile($filePath); + } catch (ParseException $e) { + $phpcsFile->addError($e->getMessage(), $stackPtr, 'Yaml'); + return $end; + } + + // Parse the environment variables. + if (isset($yaml['services']) === true) { + foreach ($yaml['services'] as $service) { + // Check if environment variables contain credentials. + if (isset($service['environment']) === true) { + foreach ($service['environment'] as $envVarName => $envVarValue) { + foreach ($checkEnvVars as $checkEnvVar) { + $envVarNameLower = strtolower($envVarName); + if (strpos($envVarNameLower, $checkEnvVar) !== false && $envVarValue !== '' && $envVarValue !== null) { + $lines = preg_grep("/($envVarName)/s", $fileContent); + $message = "Do not commit credentials in the '$fileName' file! '$envVarName' has a value. It should remain empty."; + $phpcsFile->addError($message, key($lines), 'Credentials'); + } + } + } + } + }//end foreach + }//end if + + // Only run this sniff once on the file. + return $end; + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/Generic/DeprecatedConstantsSniff.php b/phpcs/QualityAssurance/Sniffs/Generic/DeprecatedConstantsSniff.php new file mode 100644 index 000000000..2dc88e11c --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/Generic/DeprecatedConstantsSniff.php @@ -0,0 +1,80 @@ + 'Drupal::time()->getRequestTime()']; + + + /** + * Returns an array of tokens this test wants to listen for. + * + * @return array + */ + public function register() + { + return [ T_STRING ]; + + }//end register() + + + /** + * Processes this test, when one of its tokens is encountered. + * + * @param File $phpcsFile The file being scanned. + * @param int $stackPtr The position of the current token + * in the stack passed in $tokens. + * + * @return void + */ + public function process(File $phpcsFile, $stackPtr) + { + $tokens = $phpcsFile->getTokens(); + + $const = $tokens[$stackPtr]['content']; + if (array_key_exists($const, $this->deprecatedConstants) === false) { + return; + } + + $data = [$const]; + $error = 'The constant %s is deprecated'; + if (empty($this->deprecatedConstants[$const]) === false) { + $data[] = $this->deprecatedConstants[$const]; + $error .= '; use %s instead'; + } + + $phpcsFile->addError($error, $stackPtr, 'Found', $data); + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/Generic/HardcodedPathSniff.php b/phpcs/QualityAssurance/Sniffs/Generic/HardcodedPathSniff.php new file mode 100644 index 000000000..e7a10f95c --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/Generic/HardcodedPathSniff.php @@ -0,0 +1,94 @@ +getTokens(); + $end = (count($tokens) + 1); + $fileName = $phpcsFile->getFilename(); + + // Do not parse yml files. + if (strtolower(substr($fileName, -4)) === '.yml') { + return $end; + } + + // Check whole project for string "sites/". + $fileContent = file_get_contents($fileName); + if (strpos($fileContent, 'sites') === false) { + return $end; + } + + $token = $tokens[$stackPtr]; + // Path regular expression. + $regexp = 'sites/[^/]+/(files|libraries|modules|themes)'; + // If hardcoded path is found. + if (preg_match("~$regexp~", $token['content'], $matches) === 1) { + $error = "Internal hardcoded paths are not allowed. "; + switch ($matches[1]) { + case 'modules': + $error .= "Please use drupal_get_path('module', \$name)."; + break; + case 'themes': + $error .= "Please use drupal_get_path('theme', \$name)."; + break; + case 'files': + $error .= "Please use \Drupal::service('file_system')->realpath()."; + break; + } + + $phpcsFile->addError($error, $stackPtr, 'HardcodedPath'); + } + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/InfoFiles/ForbiddenSniff.php b/phpcs/QualityAssurance/Sniffs/InfoFiles/ForbiddenSniff.php new file mode 100644 index 000000000..50b2e4f57 --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/InfoFiles/ForbiddenSniff.php @@ -0,0 +1,79 @@ +getFilename()) === 1) { + // Drupal 8 style info.yml file. + $contents = file_get_contents($phpcsFile->getFilename()); + try { + $info = \Symfony\Component\Yaml\Yaml::parse($contents); + } catch (\Symfony\Component\Yaml\Exception\ParseException $e) { + // If the YAML is invalid we ignore this file. + return ($phpcsFile->numTokens + 1); + } + } else { + return ($phpcsFile->numTokens + 1); + } + + // Since we don't have forbidden properties for D8 yet we use a dummy in + // this sniff for testing purposes. Whenever we have an addition to the + // forbidden properties list we can adapt this code to reflect that + // change. + if (isset($info['no_forbidden_yet']) === true) { + $warning = 'Remove "no_forbidden_yet" from the info file.'; + $phpcsFile->addWarning($warning, $stackPtr, 'Project'); + } + + return ($phpcsFile->numTokens + 1); + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/InfoFiles/RequiredSniff.php b/phpcs/QualityAssurance/Sniffs/InfoFiles/RequiredSniff.php new file mode 100644 index 000000000..83cf6be8c --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/InfoFiles/RequiredSniff.php @@ -0,0 +1,98 @@ +getFilename(); + $fileExtension = strtolower(substr($filename, -9)); + if ($fileExtension !== '.info.yml') { + return ($phpcsFile->numTokens + 1); + } + + // Exclude config files which might contain the info.yml extension. + $filenameWithoutExtension = substr($filename, 0, -9); + if (strpos($filenameWithoutExtension, '.') !== false) { + return ($phpcsFile->numTokens + 1); + } + + $contents = file_get_contents($phpcsFile->getFilename()); + try { + $info = Yaml::parse($contents); + } catch (ParseException $e) { + // If the YAML is invalid we ignore this file. + return ($phpcsFile->numTokens + 1); + } + + if (isset($info['name']) === false) { + $error = "The key 'name' is missing in the info file"; + $phpcsFile->addError($error, $stackPtr, 'INFO'); + } + + if (isset($info['type']) === false) { + $error = "The key 'type' is missing in the info file"; + $phpcsFile->addError($error, $stackPtr, 'INFO'); + } + + if (isset($info['core']) === false && isset($info['core_version_requirement']) === false) { + $error = "One of the keys 'core' or 'core_version_requirement' is required in the info file"; + $phpcsFile->addError($error, $stackPtr, 'INFO'); + } + + if (isset($info['core']) === true && isset($info['core_version_requirement']) === true) { + $error = "The keys 'core' and 'core_version_requirement' cannot be used together in the info file"; + $phpcsFile->addError($error, $stackPtr, 'INFO'); + } + + return ($phpcsFile->numTokens + 1); + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/InstallFiles/FunctionDeclarationsSniff.php b/phpcs/QualityAssurance/Sniffs/InstallFiles/FunctionDeclarationsSniff.php new file mode 100644 index 000000000..a2a077690 --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/InstallFiles/FunctionDeclarationsSniff.php @@ -0,0 +1,80 @@ +getFilename(), -7)); + if ($fileExtension !== 'install') { + return; + } + + $tokens = $phpcsFile->getTokens(); + $functionName = $tokens[($stackPtr + 2)]['content']; + $fileName = substr(basename($phpcsFile->getFilename()), 0, -8); + $allowedHooks = [ + $fileName.'_install', + $fileName.'_install_tasks', + $fileName.'_uninstall', + $fileName.'_enable', + $fileName.'_disable', + $fileName.'_schema', + $fileName.'_field_schema', + $fileName.'_requirements', + $fileName.'_update_last_removed', + $fileName.'_update_dependencies', + ]; + if (in_array($functionName, $allowedHooks) === false && empty(preg_match('/'.$fileName.'_update_\d{4}/', $functionName)) === true) { + $warning = 'Move the "%s" function declaration to a helper class implementing ContainerInjectionInterface. Example can be found at https://git.drupalcode.org/project/drupal/blob/8.7.5/core/profiles/demo_umami/modules/demo_umami_content/demo_umami_content.install'; + $phpcsFile->addError($warning, $stackPtr, 'NonHookFound', [$functionName, $fileName, $fileName]); + } + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/InstallFiles/HookUpdate0Sniff.php b/phpcs/QualityAssurance/Sniffs/InstallFiles/HookUpdate0Sniff.php new file mode 100644 index 000000000..17585cc4d --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/InstallFiles/HookUpdate0Sniff.php @@ -0,0 +1,68 @@ +getFilename(), -7)); + if ($fileExtension !== 'install') { + return; + } + + $tokens = $phpcsFile->getTokens(); + $functionName = $tokens[($stackPtr + 2)]['content']; + $fileName = substr(basename($phpcsFile->getFilename()), 0, -8); + if (str_starts_with($functionName, "{$fileName}_update_") === true && str_ends_with($functionName, '000') === true) { + $number = preg_replace('/[^0-9]/', '', $functionName); + $phpcsFile->addError("Update schema $number is reserved for upgrading.", $stackPtr, "Update{$number}NotAllowed"); + } + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/InstallFiles/HookUpdateNSniff.php b/phpcs/QualityAssurance/Sniffs/InstallFiles/HookUpdateNSniff.php new file mode 100644 index 000000000..ce4fa73ff --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/InstallFiles/HookUpdateNSniff.php @@ -0,0 +1,111 @@ +getFilename(), -7)); + if ($fileExtension !== 'install') { + return; + } + + $tokens = $phpcsFile->getTokens(); + $functionName = $tokens[($stackPtr + 2)]['content']; + $fileName = substr(basename($phpcsFile->getFilename()), 0, -8); + if (preg_match('/'.$fileName.'_update_\d{4}$/', $functionName) === false) { + return; + } + + $find = Tokens::$methodPrefixes; + $find[] = T_WHITESPACE; + $commentEnd = $phpcsFile->findPrevious($find, ($stackPtr - 1), null, true); + if ($tokens[$commentEnd]['code'] !== T_DOC_COMMENT_CLOSE_TAG + && $tokens[$commentEnd]['code'] !== T_COMMENT + ) { + return; + } + + if ($tokens[$commentEnd]['code'] === T_COMMENT) { + return; + } + + $commentStart = $tokens[$commentEnd]['comment_opener']; + $empty = [ + T_DOC_COMMENT_WHITESPACE, + T_DOC_COMMENT_STAR, + ]; + $short = $phpcsFile->findNext($empty, ($commentStart + 1), $commentEnd, true); + if ($short === false) { + // No content at all. + return; + } + + // Account for the fact that a short description might cover + // multiple lines. + $shortContent = $tokens[$short]['content']; + $shortEnd = $short; + for ($i = ($short + 1); $i < $commentEnd; $i++) { + if ($tokens[$i]['code'] === T_DOC_COMMENT_STRING) { + if ($tokens[$i]['line'] === ($tokens[$shortEnd]['line'] + 1)) { + $shortContent .= $tokens[$i]['content']; + $shortEnd = $i; + } else { + break; + } + } + } + + // Check if hook_update_N implementation doc is formated correctly. + if ($shortContent === 'Implements hook_update_N();') { + $phpcsFile->addError('Replace "'.$shortContent.'" with a short description on the hooks functionality.', $short, "UpdateNDescription"); + }//end if + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Sniffs/InstallFiles/InstallUpdateCallbacksSniff.php b/phpcs/QualityAssurance/Sniffs/InstallFiles/InstallUpdateCallbacksSniff.php new file mode 100644 index 000000000..592671d50 --- /dev/null +++ b/phpcs/QualityAssurance/Sniffs/InstallFiles/InstallUpdateCallbacksSniff.php @@ -0,0 +1,95 @@ +getFilename(), -7)); + if ($fileExtension !== 'install') { + return; + } + + $tokens = $phpcsFile->getTokens(); + $functionName = $tokens[($stackPtr + 2)]['content']; + $fileName = substr(basename($phpcsFile->getFilename()), 0, -8); + if ($functionName !== $fileName.'_install') { + return; + } + + $string = $phpcsFile->findNext( + T_STRING, + $tokens[$stackPtr]['scope_opener'], + $tokens[$stackPtr]['scope_closer'] + ); + while ($string !== false) { + if (empty(preg_match('/^'.$fileName.'_update_\d{4}$/', $tokens[$string]['content'])) === false) { + $opener = $phpcsFile->findNext( + Tokens::$emptyTokens, + ($string + 1), + null, + true + ); + if ($opener !== false + && $tokens[$opener]['code'] === T_OPEN_PARENTHESIS + ) { + $warning = 'Do not call "%s()" in hook_install().'; + $phpcsFile->addWarning($warning, $string, 'CallbackNotAllowed', [$tokens[$string]['content']]); + } + } + + $string = $phpcsFile->findNext( + T_STRING, + ($string + 1), + $tokens[$stackPtr]['scope_closer'] + ); + }//end while + + }//end process() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/CoderSniffUnitTest.php b/phpcs/QualityAssurance/Test/CoderSniffUnitTest.php new file mode 100644 index 000000000..4a1d6b669 --- /dev/null +++ b/phpcs/QualityAssurance/Test/CoderSniffUnitTest.php @@ -0,0 +1,482 @@ + + * @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600) + * @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace QualityAssurance\Test; + +use PHP_CodeSniffer\Config; +use PHP_CodeSniffer\Files\LocalFile; +use PHP_CodeSniffer\Ruleset; +use PHP_CodeSniffer\RuntimeException; +use PHP_CodeSniffer\Util\Common; +use PHP_CodeSniffer\Util\Tokens; +use PHPUnit\Framework\TestCase; + +abstract class CoderSniffUnitTest extends TestCase +{ + + /** + * Enable or disable the backup and restoration of the $GLOBALS array. + * Overwrite this attribute in a child class of TestCase. + * Setting this attribute in setUp() has no effect! + * + * @var boolean + */ + protected $backupGlobals = false; + + /** + * The path to the standard's main directory. + * + * @var string + */ + public $standardsDir = null; + + /** + * The path to the standard's test directory. + * + * @var string + */ + public $testsDir = null; + + + /** + * Sets up this unit test. + * + * @return void + */ + protected function setUp(): void + { + $this->rootDir = __DIR__.'/../../'; + $this->testsDir = __DIR__.'/'; + $this->standardsDir = __DIR__.'/../'; + // Required to pull in all the defines from the tokens file. + $tokens = new Tokens(); + if (defined('PHP_CODESNIFFER_VERBOSITY') === false) { + define('PHP_CODESNIFFER_VERBOSITY', 0); + } + + if (defined('PHP_CODESNIFFER_CBF') === false) { + define('PHP_CODESNIFFER_CBF', 0); + } + + }//end setUp() + + + /** + * Get a list of all test files to check. + * + * These will have the same base as the sniff name but different extensions. + * We ignore the .php file as it is the class. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return string[] + */ + protected function getTestFiles($testFileBase) + { + $testFiles = []; + + $dir = substr($testFileBase, 0, strrpos($testFileBase, DIRECTORY_SEPARATOR)); + $di = new \DirectoryIterator($dir); + + foreach ($di as $file) { + $path = $file->getPathname(); + if (substr($path, 0, strlen($testFileBase)) === $testFileBase) { + if ($path !== $testFileBase.'php' && substr($path, -5) !== 'fixed') { + $testFiles[] = $path; + } + } + } + + // Put them in order. + sort($testFiles); + + return $testFiles; + + }//end getTestFiles() + + + /** + * Should this test be skipped for some reason. + * + * @return void + */ + protected function shouldSkipTest() + { + return false; + + }//end shouldSkipTest() + + + /** + * Tests the extending classes Sniff class. + * + * @preserveGlobalState disabled + * + * @return void + * @throws PHPUnit_Framework_Error + */ + final public function testSniff() + { + // Skip this test if we can't run in this environment. + if ($this->shouldSkipTest() === true) { + $this->markTestSkipped(); + } + + // Simulate an assertion. + $this->assertEquals('1', '1'); + + $sniffCode = Common::getSniffCode(get_class($this)); + list($standardName, $categoryName, $sniffName) = explode('.', $sniffCode); + + // In the case where we are running all the sniffs, the standard will + // be the root class name. + if ($this->allSniffCodes() !== false) { + list($standardName) = explode('\\', get_class($this)); + } + + $testFileBase = $this->rootDir.$standardName.DIRECTORY_SEPARATOR.'Test'.DIRECTORY_SEPARATOR.$categoryName.DIRECTORY_SEPARATOR.$sniffName.'UnitTest.'; + + $this->standardsDir = $this->rootDir.$standardName.DIRECTORY_SEPARATOR; + // $testFileBase = $this->testsDir.$categoryName.DIRECTORY_SEPARATOR.$sniffName.'UnitTest.'; + // Get a list of all test files to check. + $testFiles = $this->getTestFiles($testFileBase); + + $config = new Config(); + $config->cache = false; + $GLOBALS['PHP_CODESNIFFER_CONFIG'] = $config; + + // $config->standards = array($standardName); + $config->ignored = []; + $config->standards = [$this->standardsDir]; + + $failureMessages = []; + foreach ($testFiles as $testFile) { + // Setup to test the selected Sniff. + if ($this->allSniffCodes() !== false) { + $config->sniffs = []; + } else { + $config->sniffs = [$sniffCode]; + } + + $ruleset = new Ruleset($config); + $GLOBALS['PHP_CODESNIFFER_RULESET'] = $ruleset; + + $filename = basename($testFile); + $oldConfig = $config->getSettings(); + + try { + $this->setCliValues($filename, $config); + $phpcsFile = new LocalFile($testFile, $ruleset, $config); + $phpcsFile->process(); + } catch (RuntimeException $e) { + $this->fail('An unexpected exception has been caught: '.$e->getMessage()); + } + + $failures = $this->generateFailureMessages($phpcsFile); + $failureMessages = array_merge($failureMessages, $failures); + + if ($phpcsFile->getFixableCount() > 0) { + // Reinit to use all sniffs. + $config->sniffs = []; + $ruleset = new Ruleset($config); + try { + $this->setCliValues($filename, $config); + $phpcsFile = new LocalFile($testFile, $ruleset, $config); + $phpcsFile->process(); + } catch (RuntimeException $e) { + $this->fail('An unexpected exception has been caught: '.$e->getMessage()); + } + + // Attempt to fix the errors. + $phpcsFile->fixer->fixFile(); + $fixable = $phpcsFile->getFixableCount(); + if ($fixable > 0) { + $failureMessages[] = "Failed to fix $fixable fixable violations in $filename"; + } + + // Check for a .fixed file to check for accuracy of fixes. + $fixedFile = $testFile.'.fixed'; + if (file_exists($fixedFile) === true) { + $diff = $phpcsFile->fixer->generateDiff($fixedFile); + if (trim($diff) !== '') { + $filename = basename($testFile); + $fixedFilename = basename($fixedFile); + $failureMessages[] = "Fixed version of $filename does not match expected version in $fixedFilename; the diff is\n$diff"; + } + } + }//end if + + // Restore the config. + $config->setSettings($oldConfig); + }//end foreach + + if (empty($failureMessages) === false) { + $this->fail(implode(PHP_EOL, $failureMessages)); + } + + }//end testSniff() + + + /** + * Generate a list of test failures for a given sniffed file. + * + * @param PHP_CodeSniffer_File $file The file being tested. + * + * @return array + * @throws PHP_CodeSniffer_Exception + */ + public function generateFailureMessages(LocalFile $file) + { + $testFile = $file->getFilename(); + + $foundErrors = $file->getErrors(); + $foundWarnings = $file->getWarnings(); + $expectedErrors = $this->getErrorList(basename($testFile)); + $expectedWarnings = $this->getWarningList(basename($testFile)); + + if (is_array($expectedErrors) === false) { + throw new RuntimeException('getErrorList() must return an array'); + } + + if (is_array($expectedWarnings) === false) { + throw new RuntimeException('getWarningList() must return an array'); + } + + /* + We merge errors and warnings together to make it easier + to iterate over them and produce the errors string. In this way, + we can report on errors and warnings in the same line even though + it's not really structured to allow that. + */ + + $allProblems = []; + $failureMessages = []; + $GLOBALS['PHP_CODESNIFFER_SNIFF_CODES'] = []; + $GLOBALS['PHP_CODESNIFFER_FIXABLE_CODES'] = []; + + foreach ($foundErrors as $line => $lineErrors) { + foreach ($lineErrors as $column => $errors) { + if (isset($allProblems[$line]) === false) { + $allProblems[$line] = [ + 'expected_errors' => 0, + 'expected_warnings' => 0, + 'found_errors' => [], + 'found_warnings' => [], + ]; + } + + $foundErrorsTemp = []; + foreach ($allProblems[$line]['found_errors'] as $foundError) { + $foundErrorsTemp[] = $foundError; + } + + $errorsTemp = []; + foreach ($errors as $foundError) { + $errorsTemp[] = $foundError['message'].' ('.$foundError['source'].')'; + + $source = $foundError['source']; + if (in_array($source, $GLOBALS['PHP_CODESNIFFER_SNIFF_CODES']) === false) { + $GLOBALS['PHP_CODESNIFFER_SNIFF_CODES'][] = $source; + } + + if ($foundError['fixable'] === true + && in_array($source, $GLOBALS['PHP_CODESNIFFER_FIXABLE_CODES']) === false + ) { + $GLOBALS['PHP_CODESNIFFER_FIXABLE_CODES'][] = $source; + } + } + + $allProblems[$line]['found_errors'] = array_merge($foundErrorsTemp, $errorsTemp); + }//end foreach + + if (isset($expectedErrors[$line]) === true) { + $allProblems[$line]['expected_errors'] = $expectedErrors[$line]; + } else { + $allProblems[$line]['expected_errors'] = 0; + } + + unset($expectedErrors[$line]); + }//end foreach + + foreach ($expectedErrors as $line => $numErrors) { + if (isset($allProblems[$line]) === false) { + $allProblems[$line] = [ + 'expected_errors' => 0, + 'expected_warnings' => 0, + 'found_errors' => [], + 'found_warnings' => [], + ]; + } + + $allProblems[$line]['expected_errors'] = $numErrors; + } + + foreach ($foundWarnings as $line => $lineWarnings) { + foreach ($lineWarnings as $column => $warnings) { + if (isset($allProblems[$line]) === false) { + $allProblems[$line] = [ + 'expected_errors' => 0, + 'expected_warnings' => 0, + 'found_errors' => [], + 'found_warnings' => [], + ]; + } + + $foundWarningsTemp = []; + foreach ($allProblems[$line]['found_warnings'] as $foundWarning) { + $foundWarningsTemp[] = $foundWarning; + } + + $warningsTemp = []; + foreach ($warnings as $warning) { + $warningsTemp[] = $warning['message'].' ('.$warning['source'].')'; + } + + $allProblems[$line]['found_warnings'] = array_merge($foundWarningsTemp, $warningsTemp); + }//end foreach + + if (isset($expectedWarnings[$line]) === true) { + $allProblems[$line]['expected_warnings'] = $expectedWarnings[$line]; + } else { + $allProblems[$line]['expected_warnings'] = 0; + } + + unset($expectedWarnings[$line]); + }//end foreach + + foreach ($expectedWarnings as $line => $numWarnings) { + if (isset($allProblems[$line]) === false) { + $allProblems[$line] = [ + 'expected_errors' => 0, + 'expected_warnings' => 0, + 'found_errors' => [], + 'found_warnings' => [], + ]; + } + + $allProblems[$line]['expected_warnings'] = $numWarnings; + } + + // Order the messages by line number. + ksort($allProblems); + + foreach ($allProblems as $line => $problems) { + $numErrors = count($problems['found_errors']); + $numWarnings = count($problems['found_warnings']); + $expectedErrors = $problems['expected_errors']; + $expectedWarnings = $problems['expected_warnings']; + + $errors = ''; + $foundString = ''; + + if ($expectedErrors !== $numErrors || $expectedWarnings !== $numWarnings) { + $lineMessage = "[LINE $line]"; + $expectedMessage = 'Expected '; + $foundMessage = 'in '.basename($testFile).' but found '; + + if ($expectedErrors !== $numErrors) { + $expectedMessage .= "$expectedErrors error(s)"; + $foundMessage .= "$numErrors error(s)"; + if ($numErrors !== 0) { + $foundString .= 'error(s)'; + $errors .= implode(PHP_EOL.' -> ', $problems['found_errors']); + } + + if ($expectedWarnings !== $numWarnings) { + $expectedMessage .= ' and '; + $foundMessage .= ' and '; + if ($numWarnings !== 0) { + if ($foundString !== '') { + $foundString .= ' and '; + } + } + } + } + + if ($expectedWarnings !== $numWarnings) { + $expectedMessage .= "$expectedWarnings warning(s)"; + $foundMessage .= "$numWarnings warning(s)"; + if ($numWarnings !== 0) { + $foundString .= 'warning(s)'; + if (empty($errors) === false) { + $errors .= PHP_EOL.' -> '; + } + + $errors .= implode(PHP_EOL.' -> ', $problems['found_warnings']); + } + } + + $fullMessage = "$lineMessage $expectedMessage $foundMessage."; + if ($errors !== '') { + $fullMessage .= " The $foundString found were:".PHP_EOL." -> $errors"; + } + + $failureMessages[] = $fullMessage; + }//end if + }//end foreach + + return $failureMessages; + + }//end generateFailureMessages() + + + /** + * Get a list of CLI values to set before the file is tested. + * + * @param string $filename The name of the file being tested. + * @param \PHP_CodeSniffer\Config $config The config data for the run. + * + * @return array + */ + public function setCliValues($filename, $config) + { + return; + + }//end setCliValues() + + + /** + * Returns the lines where errors should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of errors that should occur on that line. + * + * @return array + */ + abstract protected function getErrorList(); + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + abstract protected function getWarningList(); + + + /** + * Returns a list of sniff codes that should be checked in this test. + * + * @return array The list of sniff codes. + */ + protected function allSniffCodes() + { + return false; + + }//end allSniffCodes() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/Functions/DrupalHttpRequest.inc b/phpcs/QualityAssurance/Test/Functions/DrupalHttpRequest.inc new file mode 100644 index 000000000..4bf9f03c2 --- /dev/null +++ b/phpcs/QualityAssurance/Test/Functions/DrupalHttpRequest.inc @@ -0,0 +1,12 @@ + int) + */ + public function getErrorList() + { + return [11 => 1]; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return []; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/DrupalHttpRequest.inc', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/Functions/DrupalWrappers.inc b/phpcs/QualityAssurance/Test/Functions/DrupalWrappers.inc new file mode 100644 index 000000000..33e24385c --- /dev/null +++ b/phpcs/QualityAssurance/Test/Functions/DrupalWrappers.inc @@ -0,0 +1,29 @@ + int) + */ + public function getErrorList() + { + return [ + 13 => 1, + 14 => 1, + 15 => 1, + 16 => 1, + 17 => 1, + 18 => 1, + 19 => 1, + 20 => 1, + 22 => 1, + 23 => 1, + 24 => 1, + 25 => 1, + 26 => 1, + 27 => 1, + 28 => 1, + ]; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return []; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/DrupalWrappers.inc', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/Generic/CredentialsUnitTest.php b/phpcs/QualityAssurance/Test/Generic/CredentialsUnitTest.php new file mode 100644 index 000000000..a403d2667 --- /dev/null +++ b/phpcs/QualityAssurance/Test/Generic/CredentialsUnitTest.php @@ -0,0 +1,61 @@ + int) + */ + public function getErrorList() + { + return [ + 15 => 1, + 16 => 1, + 17 => 1, + ]; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return []; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/docker-compose.yml', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/Generic/HardcodedPath.tpl.php b/phpcs/QualityAssurance/Test/Generic/HardcodedPath.tpl.php new file mode 100644 index 000000000..16ed28d06 --- /dev/null +++ b/phpcs/QualityAssurance/Test/Generic/HardcodedPath.tpl.php @@ -0,0 +1,69 @@ +subject: Block title. + * - $content: Block content. + * - $block->module: Module that generated the block. + * - $block->delta: An ID for the block, unique within each module. + * - $block->region: The block region embedding the current block. + * - $classes: String of classes that can be used to style contextually through + * CSS. It can be manipulated through the variable $classes_array from + * preprocess functions. The default values can be one or more of the + * following: + * - block: The current template type, i.e., "theming hook". + * - block-[module]: The module generating the block. For example, the user + * module is responsible for handling the default user navigation block. In + * that case the class would be 'block-user'. + * - $title_prefix (array): An array containing additional output populated by + * modules, intended to be displayed in front of the main title tag that + * appears in the template. + * - $title_suffix (array): An array containing additional output populated by + * modules, intended to be displayed after the main title tag that appears in + * the template. + * + * Helper variables: + * - $classes_array: Array of html class attribute values. It is flattened + * into a string within the variable $classes. + * - $block_zebra: Outputs 'odd' and 'even' dependent on each block region. + * - $zebra: Same output as $block_zebra but independent of any block region. + * - $block_id: Counter dependent on each block region. + * - $id: Same output as $block_id but independent of any block region. + * - $is_front: Flags true when presented in the front page. + * - $logged_in: Flags true when the current user is a logged-in member. + * - $is_admin: Flags true when the current user is an administrator. + * - $block_html_id: A valid HTML ID and guaranteed unique. + * + * @see template_preprocess() + * @see template_preprocess_block() + * @see template_process() + * + * @ingroup themeable + */ +?> +
+ + + subject): ?> +
+ subject ?> +
+ + + Hardcoded link to theme + Hardcoded link to theme + Hardcoded link to theme + Hardcoded link to theme + Hardcoded link to theme + Hardcoded link to files + + + sites\/[a-zA-Z-0-9\$\.\ \"]+\/(modules|themes|libraries)* +

This is a warning-free testing file.

+
> + Image result for google +
+ +
\ No newline at end of file diff --git a/phpcs/QualityAssurance/Test/Generic/HardcodedPathUnitTest.php b/phpcs/QualityAssurance/Test/Generic/HardcodedPathUnitTest.php new file mode 100644 index 000000000..2e0b89e77 --- /dev/null +++ b/phpcs/QualityAssurance/Test/Generic/HardcodedPathUnitTest.php @@ -0,0 +1,64 @@ + int) + */ + public function getErrorList() + { + return [ + 55 => 1, + 56 => 1, + 57 => 1, + 58 => 1, + 59 => 1, + 60 => 1, + ]; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return []; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/HardcodedPath.tpl.php', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/Generic/docker-compose.yml b/phpcs/QualityAssurance/Test/Generic/docker-compose.yml new file mode 100644 index 000000000..1d410f1f7 --- /dev/null +++ b/phpcs/QualityAssurance/Test/Generic/docker-compose.yml @@ -0,0 +1,61 @@ +version: '2' +services: + web: + image: registry.fpfis.eu/digit/devsecops/httpd-php-dev:7.1 + working_dir: ${PWD} + ports: + - 8080:8080 + volumes: + - ${PWD}:${PWD} # Non Mac users. + # - nfsmount:/var/www/html # Mac Users with the nfsmount volume. + environment: + DOCUMENT_ROOT: ${PWD} + # ASDA CREDENTIALS + # By leaving these variables empty Docker Compose will set them to their host values. + ASDA_USER: test + ASDA_PASSWORD: test + GITHUB_API_TOKEN: test + SM_CUSTOMER_API_KEY: test + SM_SECRET_PHRASE: + # XDEBUG CONFIGURATION + XDEBUG_CONFIG: idekey=cloud9ide remote_connect_back=0 remote_host=172.17.0.1 remote_enable=1 + PHP_IDE_CONFIG: "serverName=Docker" + # For Xdebug setup, run this command in the terminal: + # For Mac users: sudo ifconfig en0 alias 10.254.254.254 255.255.255.0 + # For Linux users: sudo ip addr add 10.254.254.254/32 dev lo label lo:1 + env_file: + - .env + mysql: + image: percona/percona-server:5.6 + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + volumes: + - mysql:/var/lib/mysql +# ports: +# - 3306:3306 + + # Spawn a Selenium server which also includes a VNC server on localhost:5900 (the password is "secret") + selenium: + image: selenium/standalone-chrome-debug:3.11 + environment: + - DISPLAY=:99 + - SE_OPTS=-debug + ports: + - 5900:5900 + expose: + - 4444 + +#### Mac users: uncomment the "volumes" key to enable the NFS file sharing. You can find more information about Docker for Mac here: https://github.com/openeuropa/openeuropa/blob/master/docs/starting/tooling.md#using-docker-on-macos + +#volumes: +# nfsmount: +# driver: local +# driver_opts: +# type: nfs +# o: addr=host.docker.internal,rw,nolock,hard,nointr,nfsvers=3 +# device: ":${PWD}/" + +#### End Mac users. + +volumes: + mysql: diff --git a/phpcs/QualityAssurance/Test/InfoFiles/Forbidden.info.yml b/phpcs/QualityAssurance/Test/InfoFiles/Forbidden.info.yml new file mode 100644 index 000000000..19e7bfe7d --- /dev/null +++ b/phpcs/QualityAssurance/Test/InfoFiles/Forbidden.info.yml @@ -0,0 +1,26 @@ +name: Hello World Module +description: Creates a page showing "Hello World". +package: Custom + +type: module +core: 8.x + +dependencies: + - drupal:link + - drupal:views + - paragraphs:paragraphs + - webform:webform (>=8.x-5.x) + +test_dependencies: + - drupal:image + +configure: hello_world.settings + +hidden: true + +no_forbidden_yet: true + +# Note: do not add the 'version' or 'project' properties yourself. +# They will be added automatically by the packager on drupal.org. +# version: 1.0 +# project: 'hello_world' diff --git a/phpcs/QualityAssurance/Test/InfoFiles/ForbiddenUnitTest.php b/phpcs/QualityAssurance/Test/InfoFiles/ForbiddenUnitTest.php new file mode 100644 index 000000000..ccbf7dd0f --- /dev/null +++ b/phpcs/QualityAssurance/Test/InfoFiles/ForbiddenUnitTest.php @@ -0,0 +1,57 @@ + int) + */ + public function getErrorList() + { + return []; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return [1 => 1]; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/Forbidden.info.yml', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/InfoFiles/Required.info.yml b/phpcs/QualityAssurance/Test/InfoFiles/Required.info.yml new file mode 100644 index 000000000..7d0bb2231 --- /dev/null +++ b/phpcs/QualityAssurance/Test/InfoFiles/Required.info.yml @@ -0,0 +1,25 @@ +name: Hello World Module +description: Creates a page showing "Hello World". +package: Custom + +type: module +core: 8.x +core_version_requirement: ^8.8 || ^9.0 + +dependencies: + - drupal:link + - drupal:views + - paragraphs:paragraphs + - webform:webform (>=8.x-5.x) + +test_dependencies: + - drupal:image + +configure: hello_world.settings + +hidden: true + +# Note: do not add the 'version' or 'project' properties yourself. +# They will be added automatically by the packager on drupal.org. +# version: 1.0 +# project: 'hello_world' diff --git a/phpcs/QualityAssurance/Test/InfoFiles/RequiredUnitTest.php b/phpcs/QualityAssurance/Test/InfoFiles/RequiredUnitTest.php new file mode 100644 index 000000000..4e16c9483 --- /dev/null +++ b/phpcs/QualityAssurance/Test/InfoFiles/RequiredUnitTest.php @@ -0,0 +1,57 @@ + int) + */ + public function getErrorList() + { + return [1 => 1]; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return []; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/Required.info.yml', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/InstallFiles/FunctionDeclarationsUnitTest.php b/phpcs/QualityAssurance/Test/InstallFiles/FunctionDeclarationsUnitTest.php new file mode 100644 index 000000000..8070ab4ca --- /dev/null +++ b/phpcs/QualityAssurance/Test/InstallFiles/FunctionDeclarationsUnitTest.php @@ -0,0 +1,57 @@ + int) + */ + public function getErrorList() + { + return [44 => 1]; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return []; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/error/FunctionDeclarationsUnitTest.install', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/InstallFiles/HookUpdate0UnitTest.php b/phpcs/QualityAssurance/Test/InstallFiles/HookUpdate0UnitTest.php new file mode 100644 index 000000000..208a2c0c4 --- /dev/null +++ b/phpcs/QualityAssurance/Test/InstallFiles/HookUpdate0UnitTest.php @@ -0,0 +1,57 @@ + int) + */ + public function getErrorList() + { + return [35 => 1]; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return []; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/error/HookUpdate0UnitTest.install', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/InstallFiles/HookUpdateNUnitTest.php b/phpcs/QualityAssurance/Test/InstallFiles/HookUpdateNUnitTest.php new file mode 100644 index 000000000..e18698de1 --- /dev/null +++ b/phpcs/QualityAssurance/Test/InstallFiles/HookUpdateNUnitTest.php @@ -0,0 +1,57 @@ + int) + */ + public function getErrorList() + { + return [23 => 1]; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return []; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/error/HookUpdateNUnitTest.install', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/InstallFiles/InstallUpdateCallbacksUnitTest.php b/phpcs/QualityAssurance/Test/InstallFiles/InstallUpdateCallbacksUnitTest.php new file mode 100644 index 000000000..a5f0f2b96 --- /dev/null +++ b/phpcs/QualityAssurance/Test/InstallFiles/InstallUpdateCallbacksUnitTest.php @@ -0,0 +1,57 @@ + int) + */ + public function getErrorList() + { + return []; + + }//end getErrorList() + + + /** + * Returns the lines where warnings should occur. + * + * The key of the array should represent the line number and the value + * should represent the number of warnings that should occur on that line. + * + * @return array(int => int) + */ + public function getWarningList() + { + return [12 => 1]; + + }//end getWarningList() + + + /** + * Returns a list of test files that should be checked. + * + * @param string $testFileBase The base path that the unit tests files will have. + * + * @return array The list of test files. + */ + protected function getTestFiles($testFileBase) + { + return [ + __DIR__.'/error/InstallUpdateCallbacksUnitTest.install', + ]; + + }//end getTestFiles() + + +}//end class diff --git a/phpcs/QualityAssurance/Test/InstallFiles/error/FunctionDeclarationsUnitTest.install b/phpcs/QualityAssurance/Test/InstallFiles/error/FunctionDeclarationsUnitTest.install new file mode 100644 index 000000000..a9317179c --- /dev/null +++ b/phpcs/QualityAssurance/Test/InstallFiles/error/FunctionDeclarationsUnitTest.install @@ -0,0 +1,46 @@ + + + + + + Drupal coding standard + + diff --git a/phpcs/QualityAssurance/Test/phpunit-bootstrap.php b/phpcs/QualityAssurance/Test/phpunit-bootstrap.php new file mode 100644 index 000000000..233c9cd21 --- /dev/null +++ b/phpcs/QualityAssurance/Test/phpunit-bootstrap.php @@ -0,0 +1,5 @@ + + + + QualityAssurance coding standard + + + + + + autoload.php + + + + * + + diff --git a/phpdoc.dist.xml b/phpdoc.dist.xml index 9302d9179..68ce5d1ad 100644 --- a/phpdoc.dist.xml +++ b/phpdoc.dist.xml @@ -9,7 +9,7 @@ docs - + latest diff --git a/resources/cspell/dictionary.txt b/resources/cspell/dictionary.txt index 7101107b8..abf269893 100644 --- a/resources/cspell/dictionary.txt +++ b/resources/cspell/dictionary.txt @@ -6,11 +6,13 @@ tljs # Misc. asda authmap +backoffice behat blackfire cicd commandfile cweagans +datasource dcweb devel dflydev @@ -19,31 +21,67 @@ Documentor dropsolid drush Dtoolkit +ecas endcode eulogin -eulogin +epoetry +externalauth ewcms ewpp ewppa EXITCODE expirable +fulltext fpfis gherkinlint grasmash grumphp gulpfile +issuetype +leaksignore +localdomain mailhog metapackage mydumper myloader nextcloud nexteuropa +Opencode openeuropa opensocial phpcbf phpro +readmore reqs robo ruleset +slowquery +subscrip webform xmlsitemap + +# SVG. +sodipodi +docname +namedview +inkscape +pagecolor +bordercolor +borderopacity +objecttolerance +gridtolerance +guidetolerance +pageopacity +pageshadow +showgrid +showguides +groupmode +nodetypes +toleft +totop +tmgmt +backtosite +Artboards + +# Drupal. +presave diff --git a/src/Mock.php b/src/Mock.php index 8e3d3bf1d..93c604dea 100644 --- a/src/Mock.php +++ b/src/Mock.php @@ -17,7 +17,7 @@ final class Mock * * @var string */ - private static string $defaultTag = '0.0.85'; + private static string $defaultTag = '0.0.103'; /** * The directory to download the mock to. diff --git a/src/TaskRunner/Commands/CodeReviewCommands.php b/src/TaskRunner/Commands/CodeReviewCommands.php index fcd21ffdd..d2e4be381 100644 --- a/src/TaskRunner/Commands/CodeReviewCommands.php +++ b/src/TaskRunner/Commands/CodeReviewCommands.php @@ -141,43 +141,54 @@ public function toolkitCodeReview(ConsoleIO $io, array $options = [ * * @command toolkit:run-tests * - * @option phpunit Execute the command toolkit:test-phpunit. - * @option behat Execute the command toolkit:run-behat. - * @option blackfire Execute the command toolkit:run-blackfire. - * @option components Execute the command toolkit:component-check. - * @option junit Whether to export results as junit. + * @option type The execution type, one of "clone" or "clean". + * @option junit Whether to export results as junit. * - * @return \Robo\ResultData - * The toolkit run-tests command status. + * @return int + * The toolkit run-tests command status of all executed commands. * * @SuppressWarnings(PHPMD.CyclomaticComplexity) * @SuppressWarnings(PHPMD.NPathComplexity) */ public function toolkitRunTests(ConsoleIO $io, array $options = [ - 'phpunit' => InputOption::VALUE_NONE, - 'behat' => InputOption::VALUE_NONE, - 'blackfire' => InputOption::VALUE_NONE, - 'components' => InputOption::VALUE_NONE, + 'type' => InputOption::VALUE_REQUIRED, + 'junit' => InputOption::VALUE_NONE, ]) { + $config = $this->getConfigValue('toolkit.run-tests', []); + $type = $options['type'] ?? $config['type'] ?? 'clone'; $tasks = [ - 'PHPUnit' => ['cmd' => 'tk-phpunit', 'exec' => $options['phpunit'] === true, 'result' => []], - 'Behat' => ['cmd' => 'tk-behat', 'exec' => $options['behat'] === true, 'result' => []], - 'Blackfire' => ['cmd' => 'tk-bfire', 'exec' => $options['blackfire'] === true, 'result' => []], - 'Components' => ['cmd' => 'tk-components', 'exec' => $options['components'] === true, 'result' => []], + 'PHPUnit' => [ + 'cmd' => 'tk-phpunit', + 'exec' => !empty($config['phpunit'][$type]), + 'result' => [], + ], + 'Behat' => [ + 'cmd' => 'tk-behat' . ($type === 'clean' ? ' --profile=clean' : ''), + 'exec' => !empty($config['behat'][$type]), + 'result' => [], + ], + 'Blackfire' => [ + 'cmd' => 'tk-bfire', + 'exec' => !empty($config['blackfire'][$type]), + 'result' => [], + ], + 'Components' => ['cmd' => 'tk-components', 'exec' => true, 'result' => []], ]; - $exit = 0; - $runAll = false; - // If no option is given, run all commands. - if (empty(array_filter(array_column($tasks, 'exec')))) { - $runAll = true; + + // Make sure to execute at least one of Behat or PHPUnit. + if (empty($tasks['PHPUnit']['exec']) && empty($tasks['Behat']['exec'])) { + $io->error('Tests are compulsory. You must enable at least one: behat or phpunit in your project configuration.'); + return ResultData::EXITCODE_ERROR; } + + $exit = 0; $run = $this->getBin('run'); foreach ($tasks as $name => &$task) { if ($this->isJunit()) { JunitXmlGenerator::addTestCase('Run Tests', $name); } - if ($runAll || $task['exec']) { + if ($task['exec']) { $code = $this->taskExec($run)->arg($task['cmd'])->run()->getExitCode(); $task['result'] = [$name => $code > 0 ? 'failed' : 'passed']; $exit += $code; @@ -202,7 +213,7 @@ public function toolkitRunTests(ConsoleIO $io, array $options = [ JunitXmlGenerator::generate('junit-run-tests.xml'); } - return new ResultData($exit); + return $exit; } } diff --git a/src/TaskRunner/Commands/ComponentCheckCommands.php b/src/TaskRunner/Commands/ComponentCheckCommands.php index 7d507ec25..7dc2b426e 100644 --- a/src/TaskRunner/Commands/ComponentCheckCommands.php +++ b/src/TaskRunner/Commands/ComponentCheckCommands.php @@ -15,6 +15,7 @@ use Robo\ResultData; use Robo\Symfony\ConsoleIO; use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Finder\Finder; use Symfony\Component\Yaml\Yaml; /** @@ -22,7 +23,6 @@ * * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) * @SuppressWarnings(PHPMD.ExcessiveClassLength) - * @SuppressWarnings(PHPMD.CouplingBetweenObjects) */ class ComponentCheckCommands extends AbstractCommands { @@ -547,29 +547,34 @@ public function componentUnsupported(ConsoleIO $io) * The check evaluation status. * * @SuppressWarnings(PHPMD.CyclomaticComplexity) + * @SuppressWarnings(PHPMD.NPathComplexity) */ public function componentEvaluation(ConsoleIO $io) { $this->io = $io; - if (!$this->loadComposerLock()) { + if (!$this->loadComposerLock() || !$this->loadWebsitePackages()) { return 1; } - if (!$this->loadWebsitePackages()) { - return 1; - } - // Get vendor list. + $toolkitRequirements = Website::requirements(); $vendorList = $toolkitRequirements['vendor_list'] ?? []; - // Proceed with 'blocker' option. Loop over the packages. $groupComponents = []; foreach ($this->composerLock['packages'] as $package) { - // Check if vendor belongs to the monitored vendor list. - if (in_array(explode('/', $package['name'])['0'], $vendorList)) { - $validateComponent = $this->validateComponent($package); - if ($validateComponent) { - $groupComponents[$validateComponent['1']][] = $validateComponent['0']; - } + $vendor = strtok($package['name'], '/'); + $isMonitoredVendor = in_array($vendor, $vendorList, true); + $isReviewedPackage = isset($this->packageReviews[$package['name']]); + + // Skip packages that don't belong to either group and not evaluated. + if (!$isMonitoredVendor && !$isReviewedPackage) { + continue; + } + + // Validate & group. + $validated = $this->validateComponent($package); + if ($validated) { + [$message, $group] = $validated; + $groupComponents[$group][] = $message; } } foreach ($groupComponents as $groupComponent => $messages) { @@ -577,7 +582,7 @@ public function componentEvaluation(ConsoleIO $io) foreach ($messages as $message) { $this->writeln($message); } - if ($groupComponent == 'Packages rejected/restricted:') { + if ($groupComponent == 'Restricted package(s):') { $this->writeln('In the case you want to use one of the modules listed as restricted, please open a ticket to Quality Assurance indicating the use case for evaluation and more information.'); } } @@ -677,14 +682,17 @@ public function componentComposer(ConsoleIO $io) return 1; } $composerJson = $this->getJson('composer.json'); - + $customTypes = [ + '/modules/custom' => 'drupal-custom-module', + '/themes/custom' => 'drupal-custom-theme', + '/profiles/custom' => 'drupal-custom-profile', + ]; // Check packages used in dev version. foreach ($this->composerLock['packages'] as $package) { - $typeBypass = in_array($package['type'], [ - 'drupal-custom-module', - 'drupal-custom-theme', - 'drupal-custom-profile', - ]); + if (empty($package['type']) || empty($package['version'])) { + continue; + } + $typeBypass = in_array($package['type'], $customTypes); if (!$typeBypass && preg_match('[^dev\-|\-dev$]', $package['version'])) { $this->composerFailed = true; $message = "Package {$package['name']}:{$package['version']} cannot be used in dev version."; @@ -701,8 +709,10 @@ public function componentComposer(ConsoleIO $io) $this->addJunitResult('Composer components', $message); } - // Enforce setting composer-exit-on-patch-failure. - if (empty($composerJson['extra']['composer-exit-on-patch-failure'])) { + // Enforce setting composer-exit-on-patch-failure if using cweagans/composer-patches version 1. + $exitOnPatchFail = !empty($composerJson['extra']['composer-exit-on-patch-failure']); + $composerPatchesVersion = ToolCommands::getPackagePropertyFromComposer('cweagans/composer-patches', 'version'); + if ($composerPatchesVersion && str_starts_with($composerPatchesVersion, '1.') && !$exitOnPatchFail) { $this->composerFailed = true; $message = "The composer property 'extra.composer-exit-on-patch-failure' must be set to true."; $this->writeln($message); @@ -747,7 +757,7 @@ public function componentComposer(ConsoleIO $io) // If the check value is found in the forbidden values, display an error message. if (in_array($check, $forbiddenValues)) { $message = sprintf($error, $check, $entryName, $forbiddenKey); - $this->io->error($message); + $this->writeln($message); $this->composerFailed = true; $this->addJunitResult('Composer components', $message); } @@ -768,7 +778,7 @@ public function componentComposer(ConsoleIO $io) ); foreach ($missingPlugins as $missingPlugin) { $message = "Plugin not installed, please remove from composer.json config.allow-plugins: $missingPlugin."; - $this->io->error($message); + $this->writeln($message); $this->composerFailed = true; $this->addJunitResult('Composer components', $message); } @@ -777,11 +787,47 @@ public function componentComposer(ConsoleIO $io) // Make sure the toolkit-composer-plugin is allowed. if (empty($composerJson['config']['allow-plugins'][Toolkit::PLUGIN])) { $message = 'Plugin ' . Toolkit::PLUGIN . ' must be allowed in the config.allow-plugins section of the composer.json.'; - $this->io->error($message); + $this->writeln($message); $this->composerFailed = true; $this->addJunitResult('Composer components', $message); } + // Custom local packages should respect type drupal-custom-*. + $customCodeFolder = $this->getConfigValue('toolkit.build.custom-code-folder'); + if (is_dir($customCodeFolder)) { + $finder = new Finder(); + $finder + ->files() + ->depth(['> 1', '< 3']) + ->in($customCodeFolder) + ->name('composer.json'); + foreach ($finder as $file) { + $pathName = $file->getPathname(); + $content = $this->getJson($pathName); + if (!empty($content['type']) && !in_array($content['type'], $customTypes)) { + $message = "The custom component $pathName has an invalid type {$content['type']}."; + $this->writeln($message); + $this->composerFailed = true; + $this->addJunitResult('Composer components', $message); + } + } + } + + // The custom packages installer-paths should respect the custom directories. + if (!empty($composerJson['extra']['installer-paths'])) { + foreach ($composerJson['extra']['installer-paths'] as $dir => $paths) { + foreach ($customTypes as $key => $value) { + $currentValue = $paths[0] ?? ''; + if (str_ends_with($dir, $key) && !str_ends_with($currentValue, $value)) { + $message = "The installer-path $key has a wrong value $currentValue, it should be $value."; + $this->writeln($message); + $this->composerFailed = true; + $this->addJunitResult('Composer components', $message); + } + } + } + } + if (!$this->composerFailed) { $this->say('Composer validation check passed.'); } @@ -1031,6 +1077,7 @@ protected function printComponentResults(ConsoleIO $io) * * @SuppressWarnings(PHPMD.CyclomaticComplexity) * @SuppressWarnings(PHPMD.NPathComplexity) + * @SuppressWarnings(PHPMD.ExcessiveMethodLength) */ protected function validateComponent(array $package) { @@ -1043,10 +1090,12 @@ protected function validateComponent(array $package) if (!empty($package['transport-options']['relative'])) { return; } + $config = $this->getConfig(); $modules = $this->packageReviews; $packageName = $package['name']; - $isRestricted = isset($modules[$packageName]['restricted_use']) && $modules[$packageName]['restricted_use'] !== '0'; + $isRestricted = ($modules[$packageName]['status'] ?? '') === 'restricted'; + $isRejected = ($modules[$packageName]['status'] ?? '') === 'rejected'; $hasBeenQaEd = isset($modules[$packageName]); $packageVersion = isset($package['extra']['drupal']['version']) ? explode('+', str_replace('8.x-', '', $package['extra']['drupal']['version']))[0] : $package['version']; @@ -1054,79 +1103,87 @@ protected function validateComponent(array $package) $packageVersion = in_array($packageVersion, $config->get('toolkit.invalid-versions')) ? $package['version'] : $packageVersion; // Define vars. - $message = false; - $messageType = false; + $message = $messageType = false; // If module was not reviewed yet. if (!$hasBeenQaEd) { $this->evaluationFailed = true; $message = "Package $packageName:$packageVersion has not been reviewed by QA."; - $messageType = 'Packages not reviewed:'; + $messageType = 'Not reviewed packages(s):'; $this->addJunitResult('Evaluation components', $message); + + return [$message, $messageType]; } - if ($hasBeenQaEd) { - // Validate package version against our constraints. - $constraints = ['whitelist' => false, 'blacklist' => true]; - foreach ($constraints as $constraint => $result) { - $constraintValue = !empty($modules[$packageName][$constraint]) ? $modules[$packageName][$constraint] : null; - try { - if (!is_null($constraintValue) && Semver::satisfies($packageVersion, $constraintValue) === $result) { - $this->evaluationFailed = true; - $message = "Package $packageName:$packageVersion does not meet the $constraint version constraint: $constraintValue."; - $messageType = "Package's version constraints:"; - $this->addJunitResult('Evaluation components', $message); - } - } catch (\UnexpectedValueException $exception) { + + // Validate package version against our constraints. + $constraints = ['whitelist' => false, 'blacklist' => true]; + foreach ($constraints as $constraint => $result) { + $constraintValue = !empty($modules[$packageName][$constraint]) ? $modules[$packageName][$constraint] : null; + try { + if (!is_null($constraintValue) && Semver::satisfies($packageVersion, $constraintValue) === $result) { $this->evaluationFailed = true; - $message = "Package $packageName:$packageVersion failed to parse $constraint version constraint: $constraintValue."; + $message = "Package $packageName:$packageVersion does not meet the $constraint version constraint: $constraintValue."; $messageType = "Package's version constraints:"; $this->addJunitResult('Evaluation components', $message); } + } catch (\UnexpectedValueException $exception) { + $this->evaluationFailed = true; + $message = "Package $packageName:$packageVersion failed to parse $constraint version constraint: $constraintValue."; + $messageType = "Package's version constraints:"; + $this->addJunitResult('Evaluation components', $message); } + } - if (empty($message) && $isRestricted) { - $projectId = $config->get('toolkit.project_id'); - // Check if the module is allowed for this project id. - $allowedInProject = in_array($projectId, array_map('trim', explode(',', $modules[$packageName]['restricted_use']))); - if ($allowedInProject) { - $message = "The package $packageName is authorised for the project $projectId"; - $messageType = 'Packages authorised:'; - } + if (empty($message) && $isRejected) { + $this->evaluationFailed = true; + $message = "Package $packageName:$packageVersion has been rejected by QA."; + $messageType = 'Rejected package(s):'; + $this->addJunitResult('Evaluation components', $message); + } - // Check if the module is allowed for this type of project. - $allowedProjectTypes = !empty($modules[$packageName]['allowed_project_types']) ? $modules[$packageName]['allowed_project_types'] : ''; - if (!$allowedInProject && !empty($allowedProjectTypes)) { - $allowedProjectTypes = array_map('trim', explode(',', $allowedProjectTypes)); - // Load the project from the website. - $project = Website::projectInformation($projectId); - if (in_array($project['type'], $allowedProjectTypes)) { - $allowedInProject = true; - $message = "The package $packageName is authorised for the type of project {$project['type']}"; - $messageType = 'Packages authorised:'; - } - } + if (empty($message) && $isRestricted) { + $projectId = $config->get('toolkit.project_id'); + // Check if the module is allowed for this project id. + $allowedInProject = in_array($projectId, array_map('trim', explode(',', $modules[$packageName]['restricted_use']))); + if ($allowedInProject) { + $message = "The package $packageName is authorised for the project $projectId"; + $messageType = 'Authorised package(s):'; + } - // Check if the module is allowed for this profile. - $allowedProfiles = !empty($modules[$packageName]['allowed_profiles']) ? $modules[$packageName]['allowed_profiles'] : ''; - if (!$allowedInProject && !empty($allowedProfiles)) { - $allowedProfiles = array_map('trim', explode(',', $allowedProfiles)); - $profile = $this->getProjectProfile($projectId); - if (in_array($profile, $allowedProfiles)) { - $allowedInProject = true; - $message = "The package $packageName is authorised for the profile $profile"; - $messageType = 'Packages authorised:'; - } + // Check if the module is allowed for this type of project. + $allowedProjectTypes = !empty($modules[$packageName]['allowed_project_types']) ? $modules[$packageName]['allowed_project_types'] : ''; + if (!$allowedInProject && !empty($allowedProjectTypes)) { + $allowedProjectTypes = array_map('trim', explode(',', $allowedProjectTypes)); + // Load the project from the website. + $project = Website::projectInformation($projectId); + if (in_array($project['type'], $allowedProjectTypes)) { + $allowedInProject = true; + $message = "The package $packageName is authorised for the type of project {$project['type']}"; + $messageType = 'Authorised package(s):'; } + } - // If module was not allowed in project. - if (!$allowedInProject) { - $this->evaluationFailed = true; - $message = "The use of $packageName:$packageVersion is {$modules[$packageName]['status']}."; - $messageType = 'Packages rejected/restricted:'; - $this->addJunitResult('Evaluation components', $message); + // Check if the module is allowed for this profile. + $allowedProfiles = !empty($modules[$packageName]['allowed_profiles']) ? $modules[$packageName]['allowed_profiles'] : ''; + if (!$allowedInProject && !empty($allowedProfiles)) { + $allowedProfiles = array_map('trim', explode(',', $allowedProfiles)); + $profile = $this->getProjectProfile($projectId); + if (in_array($profile, $allowedProfiles)) { + $allowedInProject = true; + $message = "The package $packageName is authorised for the profile $profile"; + $messageType = 'Authorised package(s):'; } } + + // If module was not allowed in project. + if (!$allowedInProject) { + $this->evaluationFailed = true; + $message = "The use of $packageName:$packageVersion is {$modules[$packageName]['status']}."; + $messageType = 'Restricted package(s):'; + $this->addJunitResult('Evaluation components', $message); + } } + if ($message && $messageType) { return [$message, $messageType]; } @@ -1135,7 +1192,7 @@ protected function validateComponent(array $package) /** * Loads the composer lock packages. */ - protected function loadComposerLock(): bool + private function loadComposerLock(): bool { if (!empty($this->composerLock['packages'])) { return true; @@ -1151,7 +1208,7 @@ protected function loadComposerLock(): bool /** * Loads the composer outdated results. */ - protected function loadComposerOutdated(): bool + private function loadComposerOutdated(): bool { if (!empty($this->composerOutdated)) { return true; @@ -1166,7 +1223,7 @@ protected function loadComposerOutdated(): bool /** * Loads the packages from the website. */ - protected function loadWebsitePackages(): bool + private function loadWebsitePackages(): bool { if (!empty($this->packageReviews)) { return true; @@ -1253,7 +1310,7 @@ private function getRecommendedWarningMessage(): string * @param bool $value * The value to check. */ - protected function getFailedOrPassed(bool $value): string + private function getFailedOrPassed(bool $value): string { return $value ? 'failed' : 'passed'; } @@ -1323,12 +1380,12 @@ private function restoreConfigReadOnly() * @return array * The opts.yml file content. */ - protected function getOptsYml(): array + private function getOptsYml(): array { if (isset($this->optsYml)) { return $this->optsYml; } - return ToolCommands::parseOptsYml() ?: []; + return ToolCommands::parseOptsYml(); } /** @@ -1345,7 +1402,7 @@ protected function getOptsYml(): array * * @see JunitXmlGenerator::addResult() */ - protected function addJunitResult(string $testCase, string $message, string $type = 'error'): void + private function addJunitResult(string $testCase, string $message, string $type = 'error'): void { // Skip if no junit option is used. if (!$this->isJunit()) { diff --git a/src/TaskRunner/Commands/DumpCommands.php b/src/TaskRunner/Commands/DumpCommands.php index e8ad6f8bd..7a5649dbd 100644 --- a/src/TaskRunner/Commands/DumpCommands.php +++ b/src/TaskRunner/Commands/DumpCommands.php @@ -304,6 +304,9 @@ private function nextcloudDownloadDump(ConsoleIO $io, array $options) * * @return int * The file download from custom server status. + * + * @SuppressWarnings(PHPMD.CyclomaticComplexity) + * @SuppressWarnings(PHPMD.NPathComplexity) */ private function customDownloadDump(ConsoleIO $io, array $options) { @@ -354,8 +357,23 @@ private function customDownloadDump(ConsoleIO $io, array $options) // Download the file. $show = $this->getConfigValue('toolkit.clone.show_progress', false); - $this->wgetDownloadFile($tmpFile, $destination, '.sql.gz,.tar.gz,.tar', !$show) - ->run(); + $maxRetries = (int) $this->getConfigValue('toolkit.clone.download_retries', 3); + + for ($attempt = 1; $attempt <= $maxRetries; $attempt++) { + if ($attempt > 1) { + $this->say("Retrying download (attempt $attempt of $maxRetries)..."); + if (file_exists($destination)) { + $this->taskExec('rm')->arg($destination) + ->setVerbosityThreshold(VerbosityThresholdInterface::VERBOSITY_DEBUG) + ->run(); + } + } + $this->wgetDownloadFile($tmpFile, $destination, '.sql.gz,.tar.gz,.tar', !$show) + ->run(); + if (file_exists($destination) && filesize($destination) > 0) { + break; + } + } // Remove temporary file. $this->taskExec('rm')->arg($tmpFile) @@ -431,7 +449,7 @@ private function nextcloudCheckNewerDump(string $link, string $service, string $ $link .= "/$service"; // Download the .sha file. $latest = $this->downloadShaFile($link, $tmpFolder, $service, $projectId); - $sha1 = trim(explode(' ', $latest)[0]); + $sha1 = $this->parseLatestShaData($latest, $service)['sha1']; // Compare with the local dump. if ($sha1 !== sha1_file($dump)) { @@ -459,7 +477,12 @@ private function asdaProcessFile(string $link, string $service, string $projectI { // Download the .sha file. $latest = $this->downloadShaFile($link, $tmpFolder, $service, $projectId); - $filename = trim(explode(' ', $latest)[1]); + $latestData = $this->parseLatestShaData($latest, $service); + $filename = $latestData['filename']; + + if ($filename === '') { + throw new \RuntimeException("Failed parsing filename for service '$service'."); + } // Display information about ASDA creation date. $output = strtoupper($service) . ' DATE: ' . $this->getAsdaDate($filename); @@ -469,10 +492,88 @@ private function asdaProcessFile(string $link, string $service, string $projectI // Download the database file. $this->wgetGenerateInputFile("$link/$filename", "$tmpFolder/$service.txt", true); $extension = str_ends_with($filename, '.gz') ? 'gz' : 'tar'; + $destination = "$tmpFolder/$service.$extension"; $show = $this->getConfigValue('toolkit.clone.show_progress', false); - $result = $this->wgetDownloadFile("$tmpFolder/$service.txt", "$tmpFolder/$service.$extension", '.sql.gz,.tar.gz,.tar', !$show) - ->run(); - $this->handleWgetErrors($result, $projectId); + $maxRetries = (int) $this->getConfigValue('toolkit.clone.download_retries', 3); + + $lastException = null; + for ($attempt = 1; $attempt <= $maxRetries; $attempt++) { + if ($attempt > 1) { + $this->say("Retrying download (attempt $attempt of $maxRetries)..."); + if (file_exists($destination)) { + @unlink($destination); + } + } + try { + $result = $this->wgetDownloadFile("$tmpFolder/$service.txt", $destination, '.sql.gz,.tar.gz,.tar', !$show) + ->run(); + $this->handleWgetErrors($result, $projectId); + $this->verifyDownloadedFileSha($latestData['sha1'], $destination, $service); + $lastException = null; + break; + } catch (\RuntimeException $e) { + $lastException = $e; + $this->say('Download failed: ' . $e->getMessage()); + } + } + + if ($lastException !== null) { + throw $lastException; + } + } + + /** + * Parse Nextcloud checksum file content. + * + * @param string $latest + * The checksum file content. + * @param string $service + * The service to use. + * + * @return array{sha1: string, filename: string} + * Parsed checksum data. + */ + private function parseLatestShaData(string $latest, string $service): array + { + $parts = preg_split('/\s+/', trim($latest), 2) ?: []; + $sha1 = $parts[0] ?? ''; + $filename = $parts[1] ?? ''; + if ($sha1 === '') { + throw new \RuntimeException("Failed parsing checksum for service '$service'."); + } + return [ + 'sha1' => $sha1, + 'filename' => $filename, + ]; + } + + /** + * Verify checksum for a downloaded dump file. + * + * @param string $expectedSha1 + * The expected sha1 checksum from latest.sh1. + * @param string $dumpFile + * The downloaded dump file path. + * @param string $service + * The service to use. + * + * @return void + * Throws if checksum mismatch is detected. + */ + private function verifyDownloadedFileSha(string $expectedSha1, string $dumpFile, string $service): void + { + if ($this->isSimulating()) { + return; + } + if (!file_exists($dumpFile) || filesize($dumpFile) === 0) { + throw new \RuntimeException("Downloaded dump file '$dumpFile' for service '$service' is missing or empty."); + } + + $actualSha1 = sha1_file($dumpFile); + if ($actualSha1 === false || $actualSha1 !== $expectedSha1) { + @unlink($dumpFile); + throw new \RuntimeException("Checksum mismatch for service '$service'. Expected '$expectedSha1', got '" . ($actualSha1 ?: 'N/A') . "'."); + } } /** diff --git a/src/TaskRunner/Commands/GitleaksCommands.php b/src/TaskRunner/Commands/GitleaksCommands.php index 74f78176e..bc5d16f5d 100644 --- a/src/TaskRunner/Commands/GitleaksCommands.php +++ b/src/TaskRunner/Commands/GitleaksCommands.php @@ -7,6 +7,8 @@ use Composer\Semver\Semver; use EcEuropa\Toolkit\TaskRunner\AbstractCommands; use EcEuropa\Toolkit\Toolkit; +use EcEuropa\Toolkit\Website; +use Robo\Contract\VerbosityThresholdInterface; use Robo\ResultData; use Robo\Symfony\ConsoleIO; use Symfony\Component\Console\Input\InputOption; @@ -20,6 +22,7 @@ class GitleaksCommands extends AbstractCommands protected string $repo; protected string $tag; protected string $os; + protected $io; /** * {@inheritdoc} @@ -37,29 +40,100 @@ public function getConfigurationFile() * * @command toolkit:run-gitleaks * - * @option tag The release tag of Gitleaks. - * @option os The current OS version. - * @option options The options to use when executing gitleaks command. - * - * @return int|\Robo\Collection\CollectionBuilder - * The object collection builder or integer if failed. + * @option tag The release tag of Gitleaks. + * @option sha256 SHA-256 checksum of the release archive. + * @option os The current OS version. + * @option options The options to use when executing gitleaks command. + * @option skip-dist Skip the creation of distribution. + * @option ignore-file Path to .leaksignore file (default ".leaksignore"). + * @option report-to-file Save the findings to a file. * * @aliases tk-gitleaks + * + * @usage --skip-dist --report-to-file + * + * @return int + * Return 1 if there're findings, 0 if no issues detected. */ public function toolkitRunGitleaks(ConsoleIO $io, array $options = [ 'tag' => InputOption::VALUE_REQUIRED, + 'sha256' => InputOption::VALUE_REQUIRED, 'os' => InputOption::VALUE_REQUIRED, 'options' => InputOption::VALUE_REQUIRED, + 'skip-dist' => InputOption::VALUE_NONE, + 'ignore-file' => InputOption::VALUE_REQUIRED, + 'report-to-file' => InputOption::VALUE_NONE, ]) { + $this->io = $io; $repo = $this->getConfig()->get('gitleaks.repo'); - if (!$this->download($repo, $options['tag'], $options['os'])) { + if (!$this->download($repo, $options['tag'], $options['os'], $options['sha256'])) { $io->error('Fail to download Gitleaks binary.'); return ResultData::EXITCODE_ERROR; } - $command = 'detect'; - $optionsExploded = array_filter(explode(' ', $options['options'])); + $command = $this->prepareCommand($options); + + $reportFile = 'gitleaks-report.json'; + $dist = $this->getConfigValue('toolkit.build.dist.root'); + + // Prepare the distribution and scan it. + if ($options['skip-dist'] === true) { + if (!is_dir($dist)) { + $io->error("Attempt to use --skip-dist while '$dist' is not found."); + // Consider to force build-dist instead of fail. + return ResultData::EXITCODE_ERROR; + } + $io->writeln('Skip create distribution.'); + } else { + $this->_exec($this->getBin('run') . ' toolkit:build-dist'); + } + + $io->say($command); + $this->taskExec($command)->setVerbosityThreshold(VerbosityThresholdInterface::VERBOSITY_DEBUG)->run(); + + if (!file_exists($reportFile) || empty($reportFileContent = file_get_contents($reportFile))) { + $io->error('Could not generate the report file.'); + return ResultData::EXITCODE_ERROR; + } + + if (empty($findings = json_decode($reportFileContent, true))) { + $this->printReport(['found' => 0, 'ignored' => 0]); + return ResultData::EXITCODE_OK; + } + + $report = [ + 'found' => count($findings), + 'ignored' => $this->processFindings($findings, $options), + ]; + + // Delete the report file if report-to-file is not used or if in CI. + if (Toolkit::isCiCd() || empty($options['report-to-file'])) { + unlink($reportFile); + } else { + file_put_contents($reportFile, json_encode($findings, JSON_PRETTY_PRINT)); + } + + $this->printReport($report, $findings); + return ResultData::EXITCODE_OK; + } + + /** + * Prepares the full gitleaks command including the options. + * + * @param array $options + * The command options. + * + * @return string + * The final command to execute. + */ + private function prepareCommand(array $options): string + { + $reportFile = 'gitleaks-report.json'; + $dist = $this->getConfigValue('toolkit.build.dist.root'); + $optionsExploded = array_filter(explode(' ', $options['options'] ?? [])); + + $command = "detect --source=$dist"; // Detect newer versions of gitleaks and adapt command and options. // @see https://github.com/gitleaks/gitleaks?tab=readme-ov-file#commands @@ -68,12 +142,117 @@ public function toolkitRunGitleaks(ConsoleIO $io, array $options = [ if (($index = array_search('--no-git', $optionsExploded)) !== false) { unset($optionsExploded[$index]); } - // Change the command from 'detect' to 'directory'. - $command = 'directory'; + // Change the command from 'detect' to 'directory' and remove + // --source option. + $command = "directory $dist"; } - $options['options'] = implode(' ', $optionsExploded); - return $this->taskExec($this->getBin('gitleaks') . ' ' . $command . ' ' . $options['options']); + // Make sure that redact option is not used, we will hide the secrets later on. + if (($index = array_search('--redact', $optionsExploded)) !== false) { + unset($optionsExploded[$index]); + } + + // Report to a file. + $optionsExploded[] = "--report-path=$reportFile"; + + return $this->getBin('gitleaks') . ' ' . $command . ' ' . implode(' ', $optionsExploded); + } + + /** + * Returns the ignores to use from website and project. + * + * @param array $options + * The command options. + * + * @return string[] + * The ignores to use. + */ + private function getIgnores(array $options): array + { + $ignores = []; + try { + $ignores = Website::gitleaksIgnores() ?: []; + } catch (\Exception) { + $this->io->writeln('Endpoint not available, skipping global ignores.'); + } + // Add the project ignores. + if (file_exists($options['ignore-file'])) { + $projectIgnores = file($options['ignore-file'], FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + $ignores = array_merge($ignores, $projectIgnores); + } + return $ignores; + } + + /** + * Process the findings and returns the number of ignored. + * + * @param array $findings + * The gitleaks findings. + * @param array $options + * The command options. + * + * @return int + * The number of ignored findings. + */ + private function processFindings(array &$findings, array $options): int + { + $ignored = 0; + $ignores = $this->getIgnores($options); + $dist = $this->getConfigValue('toolkit.build.dist.root'); + foreach ($findings as $index => &$finding) { + // Remove dist part from the paths. + $finding['File'] = ltrim(str_replace("$dist/", '', $finding['File']), '/'); + $finding['Fingerprint'] = ltrim(str_replace("$dist/", '', $finding['Fingerprint']), '/'); + // Build the fingerprint using the relative path to the file, + // the secret found and the rule id. + $finding['ToolkitFingerprint'] = hash('sha256', "{$finding['File']}:{$finding['Secret']}:{$finding['RuleID']}"); + + if (!empty($ignores)) { + if (in_array($finding['ToolkitFingerprint'], $ignores) || in_array($finding['Fingerprint'], $ignores)) { + unset($findings[$index]); + $ignored++; + continue; + } + } + + // Make sure that the Secret is redacted. + $finding['Match'] = str_replace($finding['Secret'], 'REDACTED', $finding['Match']); + $finding['Secret'] = 'REDACTED'; + } + + return $ignored; + } + + /** + * Prints the report information. + * + * @param array $report + * The report to print. + * @param array $findings + * The gitleaks findings to print. + */ + private function printReport(array $report, array $findings = []): void + { + if (!empty($findings)) { + $fields = $this->getConfigValue('gitleaks.fields', []); + $this->io->title('Findings:'); + foreach ($findings as $item) { + $this->io->title($item['Description']); + $maxLen = max(array_map(fn ($f) => strlen("$f:") + 2, $fields)); + foreach ($fields as $field) { + if (isset($item[$field])) { + $this->io->writeln(sprintf("%-{$maxLen}s %s", "$field:", $item[$field])); + } + } + $this->io->newLine(); + } + } + + $this->io->definitionList( + ['Findings Found' => $report['found']], + ['Findings Ignored' => $report['ignored']], + ['Findings Active' => $report['found'] - $report['ignored']], + ); } /** @@ -85,16 +264,23 @@ public function toolkitRunGitleaks(ConsoleIO $io, array $options = [ * The release tag to download. * @param string $os * The Operating system to use to download. + * @param string $sha256 + * Expected SHA-256 checksum of the downloaded archive. */ - private function download(string $repo, string $tag, string $os): bool + private function download(string $repo, string $tag, string $os, string $sha256): bool { - $link = "$repo/releases/download/v$tag/gitleaks_{$tag}_$os.tar.gz"; - $this->writeln("Downloading from $link"); if (file_exists($this->getBinPath('gitleaks')) || $this->isSimulating()) { + $this->writeln('Skip download, binary was found.'); return true; } + $link = "$repo/releases/download/v$tag/gitleaks_{$tag}_$os.tar.gz"; + $this->writeln("Downloading from $link"); $tmp = 'gitleaks_tmp'; if ($file = file_get_contents($link)) { + if (!$this->isChecksumValid($file, $sha256)) { + $this->writeln('Invalid SHA-256 checksum for downloaded Gitleaks archive.'); + return false; + } if (!file_exists($tmp)) { $this->_mkdir($tmp); } @@ -112,4 +298,17 @@ private function download(string $repo, string $tag, string $os): bool return false; } + /** + * Validate the archive SHA-256 checksum. + * + * @param string $file + * Archive contents. + * @param string $sha256 + * Expected SHA-256 checksum. + */ + private function isChecksumValid(string $file, string $sha256): bool + { + return hash_equals(strtolower($sha256), strtolower(hash('sha256', $file))); + } + } diff --git a/src/Toolkit.php b/src/Toolkit.php index ac0da1ca8..5083b9f29 100644 --- a/src/Toolkit.php +++ b/src/Toolkit.php @@ -14,7 +14,7 @@ final class Toolkit /** * Constant holding the current version. */ - public const VERSION = '10.26.2'; + public const VERSION = '10.29.1'; /** * The Toolkit repository. diff --git a/src/Website.php b/src/Website.php index f10af962a..ec99c5d99 100644 --- a/src/Website.php +++ b/src/Website.php @@ -138,7 +138,6 @@ public static function get(string $url, ?AuthorizationInterface $auth = null): s if ($result === false) { throw new \Exception(sprintf('Curl request to endpoint "%s" failed.', $url)); } - curl_close($curl); return $content; } @@ -168,7 +167,6 @@ public static function getSessionToken(): string $ch = curl_init(self::url() . '/session/token'); curl_setopt_array($ch, $options); $token = (string) curl_exec($ch); - curl_close($ch); $GLOBALS['session_token'] = $token; return $token; } @@ -206,7 +204,6 @@ public static function post(array $fields, AuthorizationInterface $auth): string ]); curl_exec($ch); $code = (string) curl_getinfo($ch, CURLINFO_HTTP_CODE); - curl_close($ch); return $code; } @@ -321,6 +318,25 @@ public static function packages() return empty($data) ? false : $data; } + /** + * Returns the gitleaks ignores from the endpoint. + * + * @return false|array + * The gitleaks ignores from the endpoint. + */ + public static function gitleaksIgnores() + { + if (empty($auth = self::apiAuth())) { + return false; + } + $url = self::url() . '/api/v1/gitleaks-ignores'; + if (!empty($projectId = getenv('TOOLKIT_PROJECT_ID'))) { + $url .= '?project=' . $projectId; + } + $data = self::getWithMockFallback($url, $auth); + return empty($data) ? [] : $data['gitleaks_ignores']; + } + /** * Returns the notifications from the endpoint. * diff --git a/tests/AbstractTest.php b/tests/AbstractTest.php index 4b2398820..ef6f91025 100644 --- a/tests/AbstractTest.php +++ b/tests/AbstractTest.php @@ -241,7 +241,7 @@ protected function debugExpectations(string $content, array $expectations) */ protected function trimEachLine(string $text): string { - return trim(implode(PHP_EOL, array_map('trim', explode(PHP_EOL, $text)))); + return trim(implode(PHP_EOL, array_filter(array_map('trim', explode(PHP_EOL, $text))))); } /** diff --git a/tests/Features/Commands/GitleaksCommandsTest.php b/tests/Features/Commands/GitleaksCommandsTest.php index 648f6a8ae..355bcd76b 100644 --- a/tests/Features/Commands/GitleaksCommandsTest.php +++ b/tests/Features/Commands/GitleaksCommandsTest.php @@ -6,6 +6,7 @@ use EcEuropa\Toolkit\TaskRunner\Commands\GitleaksCommands; use EcEuropa\Toolkit\Tests\AbstractTest; +use Symfony\Component\Yaml\Yaml; /** * Test Gitleaks commands. @@ -31,13 +32,23 @@ public static function dataProvider() * * @param string $command * A command. + * @param array $configuration + * A configuration. + * @param array $resources + * Resources needed for the test. * @param array $expectations * Tests expected. * * @dataProvider dataProvider */ - public function testGitleaks(string $command, array $expectations = []) + public function testGitleaks(string $command, array $configuration = [], array $resources = [], array $expectations = []) { + // Setup configuration file. + if (!empty($configuration)) { + $this->fs->dumpFile($this->getSandboxFilepath('runner.yml'), Yaml::dump($configuration)); + } + $this->prepareResources($resources); + // Run command. $result = $this->runCommand($command); // Assert expectations. diff --git a/tests/Unit/DumpCommandsTest.php b/tests/Unit/DumpCommandsTest.php new file mode 100644 index 000000000..e411a26b1 --- /dev/null +++ b/tests/Unit/DumpCommandsTest.php @@ -0,0 +1,121 @@ +setAccessible(true); + + $result = $method->invoke( + new DumpCommands(), + "5baa8031434fb4e95a0982d65ed196271328bbe8 dump.sql.gz\n", + 'mysql' + ); + + $this->assertSame('5baa8031434fb4e95a0982d65ed196271328bbe8', $result['sha1']); + $this->assertSame('dump.sql.gz', $result['filename']); + } + + /** + * Test parsing invalid latest.sh1 content. + */ + public function testParseLatestShaDataInvalidContent(): void + { + $method = new ReflectionMethod(DumpCommands::class, 'parseLatestShaData'); + $method->setAccessible(true); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage("Failed parsing checksum for service 'mysql'."); + $method->invoke(new DumpCommands(), "\n", 'mysql'); + } + + /** + * Test checksum verification succeeds using a real fixture SHA1. + */ + public function testVerifyDownloadedFileShaRealFixtureMatch(): void + { + $command = $this->createDumpCommandsMock(false); + + $file = realpath(__DIR__ . '/../fixtures/samples/sample-dump.sql.gz'); + $this->assertNotFalse($file); + + $method = new ReflectionMethod(DumpCommands::class, 'verifyDownloadedFileSha'); + $method->setAccessible(true); + $method->invoke($command, '829c3804401b0727f70f73d4415e162400cbe57b', $file, 'mysql'); + + $this->assertFileExists($file); + } + + /** + * Test checksum verification succeeds when checksum matches. + */ + public function testVerifyDownloadedFileShaMatch(): void + { + $command = $this->createDumpCommandsMock(false); + + $file = tempnam(sys_get_temp_dir(), 'dump-sha-ok-'); + $this->assertNotFalse($file); + file_put_contents($file, 'sample dump content'); + + $method = new ReflectionMethod(DumpCommands::class, 'verifyDownloadedFileSha'); + $method->setAccessible(true); + $method->invoke($command, sha1_file($file), $file, 'mysql'); + + $this->assertFileExists($file); + @unlink($file); + } + + /** + * Test checksum verification fails when checksum does not match. + */ + public function testVerifyDownloadedFileShaMismatch(): void + { + $command = $this->createDumpCommandsMock(false); + + $file = tempnam(sys_get_temp_dir(), 'dump-sha-ko-'); + $this->assertNotFalse($file); + file_put_contents($file, 'sample dump content'); + + $method = new ReflectionMethod(DumpCommands::class, 'verifyDownloadedFileSha'); + $method->setAccessible(true); + + try { + $method->invoke($command, '0000000000000000000000000000000000000000', $file, 'mysql'); + $this->fail('Expected checksum mismatch exception was not thrown.'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString("Checksum mismatch for service 'mysql'", $e->getMessage()); + } + $this->assertFileDoesNotExist($file); + } + + /** + * Create a partial mock for DumpCommands with mocked simulation status. + */ + private function createDumpCommandsMock(bool $isSimulating): DumpCommands + { + $command = $this->getMockBuilder(DumpCommands::class) + ->onlyMethods(['isSimulating']) + ->getMock(); + $command->method('isSimulating')->willReturn($isSimulating); + return $command; + } + +} diff --git a/tests/Unit/GitleaksCommandsChecksumTest.php b/tests/Unit/GitleaksCommandsChecksumTest.php new file mode 100644 index 000000000..cd0142080 --- /dev/null +++ b/tests/Unit/GitleaksCommandsChecksumTest.php @@ -0,0 +1,40 @@ +setAccessible(true); + + $content = 'sample-archive-content'; + $validSha = hash('sha256', $content); + + $command = new GitleaksCommands(); + + $this->assertTrue($method->invoke($command, $content, $validSha)); + $this->assertFalse($method->invoke($command, $content, str_repeat('0', 64))); + $this->assertFalse($method->invoke($command, $content, '')); + } + +} diff --git a/tests/fixtures/commands/code-review.yml b/tests/fixtures/commands/code-review.yml index 19356082b..d6e8b5b25 100644 --- a/tests/fixtures/commands/code-review.yml +++ b/tests/fixtures/commands/code-review.yml @@ -52,17 +52,107 @@ - command: toolkit:run-tests configuration: [] resources: [] + expectations: + - string_contains: PHPUnit skip + - string_contains: Behat passed + - string_contains: Blackfire skip + - string_contains: Components passed + +- command: toolkit:run-tests + configuration: + toolkit: + run-tests: + phpunit: + clone: false + clean: false + behat: + clone: true + clean: true + blackfire: + clone: false + clean: false + resources: [] + expectations: + - string_contains: PHPUnit skip + - string_contains: Behat passed + - string_contains: Blackfire skip + - string_contains: Components passed + +- command: toolkit:run-tests + configuration: + toolkit: + run-tests: + phpunit: + clone: true + clean: true + behat: + clone: true + clean: true + blackfire: + clone: true + clean: true + resources: [] expectations: - string_contains: PHPUnit passed - string_contains: Behat passed - string_contains: Blackfire passed - string_contains: Components passed -- command: toolkit:run-tests --behat - configuration: [] +- command: toolkit:run-tests --type=clean + configuration: + toolkit: + run-tests: + type: clean + phpunit: + clone: false + clean: false + behat: + clone: false + clean: true + blackfire: + clone: false + clean: false resources: [] expectations: + - string_contains: --profile=clean - string_contains: PHPUnit skip - string_contains: Behat passed - string_contains: Blackfire skip - - string_contains: Components skip + - string_contains: Components passed + +- command: toolkit:run-tests + configuration: + toolkit: + run-tests: + phpunit: + clone: false + behat: + clone: false + resources: [] + expectations: + - string_contains: "Tests are compulsory. You must enable at least one: behat or phpunit" + +- command: toolkit:run-tests --type=clean + configuration: + toolkit: + run-tests: + phpunit: + clean: false + behat: + clean: false + resources: [] + expectations: + - string_contains: "Tests are compulsory. You must enable at least one: behat or phpunit" + +- command: toolkit:run-tests + configuration: + toolkit: + run-tests: + type: clean + phpunit: + clean: false + behat: + clean: false + resources: [] + expectations: + - string_contains: "Tests are compulsory. You must enable at least one: behat or phpunit" diff --git a/tests/fixtures/commands/component-check.yml b/tests/fixtures/commands/component-check.yml index ba1337b31..b1e8ac7f9 100644 --- a/tests/fixtures/commands/component-check.yml +++ b/tests/fixtures/commands/component-check.yml @@ -65,25 +65,21 @@ Package's version constraints: Package drupal/github_connect:2.0.0-alpha1 does not meet the whitelist version constraint: ^1.0. - Checking Development components. ================================ > Development components check passed. - Checking Composer components. ============================= > Composer validation check passed. - Checking Configuration components. ================================== > Project configuration check passed. - Results: ======== @@ -201,7 +197,7 @@ - string_contains: Development module check passed - string_contains: Composer validation check passed -- command: 'toolkit:component-check' +- command: 'check:evaluation' configuration: [ ] tokens: '' resources: @@ -209,9 +205,9 @@ - from: sample-internal-dependency.lock to: composer.lock expectations: - - string_contains: Evaluation module check passed + - string_contains: Evaluation module check passed -- command: 'toolkit:component-check' +- command: 'check:composer' configuration: [ ] tokens: '' resources: @@ -230,9 +226,9 @@ - string_contains: Package test/package2:1.0.0-dev cannot be used in dev version. - not_string_contains: Package test/module:1.0.0-dev cannot be used in dev version. - string_contains: Plugin ec-europa/toolkit-composer-plugin must be allowed - - string_contains: Composer validation check failed + - not_string_contains: Composer validation check passed -- command: 'toolkit:component-check' +- command: 'check:composer' configuration: [ ] tokens: '' resources: @@ -244,9 +240,9 @@ { "packages": [ { "name": "test/package", "type": "library", "version": "1.0.0" } ] } expectations: - string_contains: The composer property 'extra.enable-patching' cannot be set to true. - - string_contains: Composer validation check failed + - not_string_contains: Composer validation check passed -- command: 'toolkit:component-check' +- command: 'check:composer' configuration: [ ] tokens: '' resources: @@ -262,41 +258,69 @@ { "packages": [ { "name": "test/package", "type": "library", "version": "1.0.0" } ] } expectations: - not_string_contains: extra.enable-patching - - string_contains: Composer validation check passed + - string_contains: Composer validation check passed -- command: 'toolkit:component-check' +# test composer-exit-on-patch-failure: true. +- command: 'check:composer' configuration: [ ] tokens: '' resources: - file: composer.json content: | - { "extra": { "composer-exit-on-patch-failure": false } } + { + "extra": { "composer-exit-on-patch-failure": true }, + "config": { "allow-plugins": { "ec-europa/toolkit-composer-plugin": true } } + } - file: composer.lock content: | - { "packages": [ { "name": "test/package", "type": "library", "version": "1.0.0" } ] } + { "packages": [ + { "name": "test/package", "type": "library", "version": "1.0.0" }, + { "name": "cweagans/composer-patches", "type": "composer-plugin", "version": "1.7.3" } + ] } expectations: - - string_contains: The composer property 'extra.composer-exit-on-patch-failure' must be set to true. - - string_contains: Composer validation check failed + - not_string_contains: composer-exit-on-patch-failure + - string_contains: Composer validation check passed -- command: 'toolkit:component-check' +# test not using package. +- command: 'check:composer' configuration: [ ] tokens: '' resources: - file: composer.json content: | { - "name": "ec-europa/toolkit", - "extra": { "composer-exit-on-patch-failure": true }, - "config": { "allow-plugins": { "ec-europa/toolkit-composer-plugin": true } } + "extra": { "composer-exit-on-patch-failure": false }, + "config": { "allow-plugins": { "ec-europa/toolkit-composer-plugin": true } } } - file: composer.lock content: | { "packages": [ { "name": "test/package", "type": "library", "version": "1.0.0" } ] } expectations: - - not_string_contains: extra.composer-exit-on-patch-failure - - string_contains: Composer validation check passed + - not_string_contains: composer-exit-on-patch-failure + - string_contains: Composer validation check passed -- command: 'toolkit:component-check' +# test composer-exit-on-patch-failure should only report in version 1. +- command: 'check:composer' + configuration: [ ] + tokens: '' + resources: + - file: composer.json + content: | + { + "extra": { "composer-exit-on-patch-failure": false }, + "config": { "allow-plugins": { "ec-europa/toolkit-composer-plugin": true } } + } + - file: composer.lock + content: | + { "packages": [ + { "name": "test/package", "type": "library", "version": "1.0.0" }, + { "name": "cweagans/composer-patches", "type": "composer-plugin", "version": "2.0.0" } + ] } + expectations: + - not_string_contains: composer-exit-on-patch-failure + - string_contains: Composer validation check passed + +- command: 'check:composer' configuration: toolkit: components: @@ -329,9 +353,9 @@ - not_string_contains: The patch 'https://www.drupal.org/files/issues/2023-01-01/12345-1.patch' is not valid - not_string_contains: The patch 'https://drupal.org/files/issues/2023-01-01/12345-2.patch' is not valid - not_string_contains: The patch 'http://drupal.org/files/issues/2023-01-01/12345-3.patch' is not valid - - string_contains: Composer validation check passed + - string_contains: Composer validation check passed -- command: 'toolkit:component-check' +- command: 'check:composer' configuration: toolkit: components: @@ -365,9 +389,9 @@ - string_contains: The patch 'https://www.drupal.org/files/issues/2023-01-01/12345-1.patch' is not valid - string_contains: The patch 'https://drupal.org/files/issues/2023-01-01/12345-2.patch' is not valid - string_contains: The patch 'http://drupal.org/files/issues/2023-01-01/12345-3.patch' is not valid - - string_contains: Composer validation check failed + - not_string_contains: Composer validation check passed -- command: 'toolkit:component-check' +- command: 'check:evaluation' configuration: toolkit: project_id: digit-dqa @@ -380,7 +404,7 @@ expectations: - string_contains: The package drupal/codesnippet is authorised for the project digit-dqa -- command: 'toolkit:component-check' +- command: 'check:evaluation' configuration: [ ] tokens: '' resources: @@ -388,13 +412,13 @@ - file: composer.lock content: | { "packages": [ { "name": "drupal/pipeline", "type": "drupal-module", "version": "1.0.0" } ] } - - file: .toolkit-mock/0.0.85/api/v1/project/ec-europa/toolkit/information.json + - file: .toolkit-mock/0.0.103/api/v1/project/ec-europa/toolkit/information.json content: | [{ "name": "toolkit", "type": "Openeuropa", "environments": [{ "profile": "minimal", "type": "Production" }] }] - - file: .toolkit-mock/0.0.85/api/v1/toolkit-requirements.json + - file: .toolkit-mock/0.0.103/api/v1/toolkit-requirements.json content: | { "php_version": "8.1", "toolkit": "^3.7.2|^9.14|^10.4", "drupal": "^7.96|^10.1.4|^10.0.11", "vendor_list": [ "drupal" ] } - - file: .toolkit-mock/0.0.85/api/v1/package-reviews.json + - file: .toolkit-mock/0.0.103/api/v1/package-reviews.json content: | [ { "type": "drupal-module", @@ -421,7 +445,7 @@ expectations: - string_contains: The package drupal/pipeline is authorised for the type of project Openeuropa -- command: 'toolkit:component-check' +- command: 'check:evaluation' configuration: [ ] tokens: '' resources: @@ -429,13 +453,13 @@ - file: composer.lock content: | { "packages": [ { "name": "drupal/pipeline", "type": "drupal-module", "version": "1.0.0" } ] } - - file: .toolkit-mock/0.0.85/api/v1/project/ec-europa/toolkit/information.json + - file: .toolkit-mock/0.0.103/api/v1/project/ec-europa/toolkit/information.json content: | [{ "name": "toolkit", "type": "Openeuropa", "environments": [{ "profile": "minimal", "type": "Production" }] }] - - file: .toolkit-mock/0.0.85/api/v1/toolkit-requirements.json + - file: .toolkit-mock/0.0.103/api/v1/toolkit-requirements.json content: | { "php_version": "8.1", "toolkit": "^3.7.2|^9.14|^10.4", "drupal": "^7.96|^10.1.4|^10.0.11", "vendor_list": [ "drupal" ] } - - file: .toolkit-mock/0.0.85/api/v1/package-reviews.json + - file: .toolkit-mock/0.0.103/api/v1/package-reviews.json content: | [ { "type": "drupal-module", @@ -462,14 +486,14 @@ expectations: - string_contains: The package drupal/pipeline is authorised for the profile minimal -- command: 'toolkit:component-check' +- command: 'check:mandatory' configuration: [ ] tokens: '' resources: - file: composer.lock content: | - { "packages": [ { "name": "ec-europa/toolkit" } ] } - - file: .toolkit-mock/0.0.85/api/v1/package-reviews.json + { "packages": [ { "name": "ec-europa/toolkit", "type": "library", "version": "11.26.2" } ] } + - file: .toolkit-mock/0.0.103/api/v1/package-reviews.json content: | [ { "type": "drupal-module", @@ -488,7 +512,7 @@ - string_contains: Package mandatory is mandatory and is not present on the project - not_string_contains: Package pipeline is mandatory and is not present on the project -- command: 'toolkit:component-check' +- command: 'check:composer' configuration: toolkit: components: @@ -532,7 +556,7 @@ - string_contains: Subsite\composer\SetupWizard::setup - string_contains: scripts/composer/SetupWizard.php -- command: 'toolkit:component-check' +- command: 'check:configuration' configuration: [] tokens: '' resources: @@ -550,7 +574,7 @@ - not_string_contains: grumphp.yml.dist file was detected - string_contains: .env file was detected -- command: 'toolkit:component-check' +- command: 'check:configuration' configuration: [] tokens: '' resources: @@ -561,7 +585,7 @@ expectations: - string_contains: grumphp.yml.dist file was detected -- command: 'toolkit:component-check' +- command: 'check:configuration' configuration: [] tokens: '' resources: @@ -587,7 +611,7 @@ - string_contains: ASDA_PASSWORD - not_string_contains: QA_API_BASIC_AUTH -- command: 'toolkit:component-check' +- command: 'check:configuration' configuration: [] tokens: '' resources: @@ -611,7 +635,7 @@ - string_contains: ASDA_PASSWORD - not_string_contains: QA_API_BASIC_AUTH -- command: 'toolkit:component-check' +- command: 'check:configuration' configuration: [] tokens: '' resources: @@ -632,7 +656,7 @@ - string_contains: ASDA_PASSWORD - not_string_contains: QA_API_BASIC_AUTH -- command: 'toolkit:component-check' +- command: 'check:configuration' configuration: [] tokens: '' resources: @@ -651,7 +675,7 @@ expectations: - string_contains: When using phpstan/extension-installer you should not manually include -- command: 'toolkit:component-check' +- command: 'check:configuration' configuration: [] tokens: '' resources: @@ -666,7 +690,7 @@ expectations: - string_contains: When using phpstan/extension-installer you should not manually include -- command: 'toolkit:component-check' +- command: 'check:configuration' configuration: [] tokens: '' resources: @@ -683,7 +707,7 @@ expectations: - not_string_contains: When using phpstan/extension-installer you should not manually include -- command: 'toolkit:component-check' +- command: 'check:composer' configuration: [] tokens: '' resources: @@ -708,7 +732,7 @@ - not_string_contains: "config.allow-plugins: dealerdirect/phpcodesniffer-composer-installer" - string_contains: "config.allow-plugins: phpstan/extension-installer" -- command: 'toolkit:component-check' +- command: 'check:npm-outdated' configuration: toolkit: components: @@ -730,7 +754,7 @@ content: | { "npm_install": "True" } expectations: - - string_contains: NPM Outdated check passed + - string_contains: NPM Outdated check passed - command: 'toolkit:component-check' configuration: diff --git a/tests/fixtures/commands/configuration.yml b/tests/fixtures/commands/configuration.yml index 90097463e..a474f6f2a 100644 --- a/tests/fixtures/commands/configuration.yml +++ b/tests/fixtures/commands/configuration.yml @@ -31,6 +31,7 @@ mydumper: /scripts/backupdb.sh type: cloud show_progress: false + download_retries: 3 nextcloud: services: mysql url: nextcloud.url/files @@ -310,7 +311,7 @@ standards: - ./vendor/drupal/coder/coder_sniffer/Drupal - ./vendor/drupal/coder/coder_sniffer/DrupalPractice - - ./vendor/ec-europa/qa-automation/phpcs/QualityAssurance + - ./vendor/ec-europa/toolkit/phpcs/QualityAssurance ignore_patterns: - dist/ - .cache/ @@ -353,7 +354,7 @@ standards: - ./vendor/drupal/coder/coder_sniffer/Drupal - ./vendor/drupal/coder/coder_sniffer/DrupalPractice - - ./vendor/ec-europa/qa-automation/phpcs/QualityAssurance + - ./vendor/ec-europa/toolkit/phpcs/QualityAssurance ignore_patterns: - dist/ - .cache/ diff --git a/tests/fixtures/commands/docker.yml b/tests/fixtures/commands/docker.yml index 8b39756d5..c2f936d60 100644 --- a/tests/fixtures/commands/docker.yml +++ b/tests/fixtures/commands/docker.yml @@ -1,7 +1,7 @@ - command: 'docker:refresh-configuration' configuration: toolkit: - project_id: + project_id: ~ resources: [] expectations: - contains: The configuration toolkit.project_id value is not valid. diff --git a/tests/fixtures/commands/drupal-settings-setup.yml b/tests/fixtures/commands/drupal-settings-setup.yml index 918924811..406efc499 100644 --- a/tests/fixtures/commands/drupal-settings-setup.yml +++ b/tests/fixtures/commands/drupal-settings-setup.yml @@ -48,8 +48,6 @@ // Dropsolid requires a value to be set for max-age. $config['system.performance']['cache']['page']['max_age'] = 21600; - - // Load environment development override configuration, if available. // Keep this code block at the end of this file to take full effect. if (file_exists($app_root . '/' . $site_path . '/settings.override.php')) { @@ -176,8 +174,6 @@ // Dropsolid requires a value to be set for max-age. $config['system.performance']['cache']['page']['max_age'] = 21600; - - // Load environment development override configuration, if available. // Keep this code block at the end of this file to take full effect. if (file_exists($app_root . '/' . $site_path . '/settings.override.php')) { diff --git a/tests/fixtures/commands/drupal.yml b/tests/fixtures/commands/drupal.yml index ecd4acbfa..32a3720e8 100644 --- a/tests/fixtures/commands/drupal.yml +++ b/tests/fixtures/commands/drupal.yml @@ -336,8 +336,6 @@ Field types that should be sanitised: ===================================== - - Fields that should be sanitised by their name: ============================================== diff --git a/tests/fixtures/commands/dump.yml b/tests/fixtures/commands/dump.yml index 84de14ec7..1e5a315c1 100644 --- a/tests/fixtures/commands/dump.yml +++ b/tests/fixtures/commands/dump.yml @@ -419,6 +419,24 @@ ->option('--server-response 2>&1') ->printMetadata() [Simulator] Running wget -i ./tmp.txt -O ./mysql.gz -A '.sql.gz,.tar.gz,.tar' -P ./ --server-response 2>&1 + > Retrying download (attempt 2 of 3)... + [Simulator] Simulating Exec('wget') + ->option('-i', './tmp.txt') + ->option('-O', './mysql.gz') + ->option('-A', '.sql.gz,.tar.gz,.tar') + ->option('-P', './') + ->option('--server-response 2>&1') + ->printMetadata() + [Simulator] Running wget -i ./tmp.txt -O ./mysql.gz -A '.sql.gz,.tar.gz,.tar' -P ./ --server-response 2>&1 + > Retrying download (attempt 3 of 3)... + [Simulator] Simulating Exec('wget') + ->option('-i', './tmp.txt') + ->option('-O', './mysql.gz') + ->option('-A', '.sql.gz,.tar.gz,.tar') + ->option('-P', './') + ->option('--server-response 2>&1') + ->printMetadata() + [Simulator] Running wget -i ./tmp.txt -O ./mysql.gz -A '.sql.gz,.tar.gz,.tar' -P ./ --server-response 2>&1 [Simulator] Running rm ./tmp.txt [ERROR] Custom : Could not fetch the file example.com/mysql.gz @@ -448,6 +466,24 @@ ->option('--server-response 2>&1') ->printMetadata() [Simulator] Running wget -i ./tmp.txt -O ./dump.gz -A '.sql.gz,.tar.gz,.tar' -P ./ --server-response 2>&1 + > Retrying download (attempt 2 of 3)... + [Simulator] Simulating Exec('wget') + ->option('-i', './tmp.txt') + ->option('-O', './dump.gz') + ->option('-A', '.sql.gz,.tar.gz,.tar') + ->option('-P', './') + ->option('--server-response 2>&1') + ->printMetadata() + [Simulator] Running wget -i ./tmp.txt -O ./dump.gz -A '.sql.gz,.tar.gz,.tar' -P ./ --server-response 2>&1 + > Retrying download (attempt 3 of 3)... + [Simulator] Simulating Exec('wget') + ->option('-i', './tmp.txt') + ->option('-O', './dump.gz') + ->option('-A', '.sql.gz,.tar.gz,.tar') + ->option('-P', './') + ->option('--server-response 2>&1') + ->printMetadata() + [Simulator] Running wget -i ./tmp.txt -O ./dump.gz -A '.sql.gz,.tar.gz,.tar' -P ./ --server-response 2>&1 [Simulator] Running rm ./tmp.txt [ERROR] Custom : Could not fetch the file example.com/dump.gz @@ -470,6 +506,10 @@ [Simulator] Simulating Filesystem\FilesystemStack() > Starting download [Simulator] Running wget -i ./tmp.txt -O ./dump.gz -A '.sql.gz,.tar.gz,.tar' -P ./ --server-response 2>&1 + > Retrying download (attempt 2 of 3)... + [Simulator] Running wget -i ./tmp.txt -O ./dump.gz -A '.sql.gz,.tar.gz,.tar' -P ./ --server-response 2>&1 + > Retrying download (attempt 3 of 3)... + [Simulator] Running wget -i ./tmp.txt -O ./dump.gz -A '.sql.gz,.tar.gz,.tar' -P ./ --server-response 2>&1 [Simulator] Running rm ./tmp.txt [ERROR] Custom : Could not fetch the file example.com/dump.gz diff --git a/tests/fixtures/commands/gitleaks.yml b/tests/fixtures/commands/gitleaks.yml index 8f240a9ce..b0bee8174 100644 --- a/tests/fixtures/commands/gitleaks.yml +++ b/tests/fixtures/commands/gitleaks.yml @@ -1,20 +1,115 @@ - command: 'toolkit:run-gitleaks' + configuration: [] + resources: + - file: gitleaks-report.json + content: '[]' expectations: - contains: | - Downloading from https://github.com/gitleaks/gitleaks/releases/download/v8.27.2/gitleaks_8.27.2_linux_x64.tar.gz - [Simulator] Simulating Exec('./vendor/bin/gitleaks directory --no-banner -v --follow-symlinks') - [Simulator] Running ./vendor/bin/gitleaks directory --no-banner -v --follow-symlinks + Skip download, binary was found. + [Simulator] Simulating Exec('./vendor/bin/run toolkit:build-dist') + [Simulator] Running ./vendor/bin/run toolkit:build-dist + > ./vendor/bin/gitleaks directory dist --no-banner -v --follow-symlinks --report-path=gitleaks-report.json + [Simulator] Running ./vendor/bin/gitleaks directory dist --no-banner -v --follow-symlinks --report-path=gitleaks-report.json + ------------------ --- + Findings Found 0 + Findings Ignored 0 + Findings Active 0 + ------------------ --- - command: 'toolkit:run-gitleaks --options="--no-banner --redact"' + configuration: [] + resources: + - file: gitleaks-report.json + content: '[]' expectations: - contains: | - Downloading from https://github.com/gitleaks/gitleaks/releases/download/v8.27.2/gitleaks_8.27.2_linux_x64.tar.gz - [Simulator] Simulating Exec('./vendor/bin/gitleaks directory --no-banner --redact') - [Simulator] Running ./vendor/bin/gitleaks directory --no-banner --redact + Skip download, binary was found. + [Simulator] Simulating Exec('./vendor/bin/run toolkit:build-dist') + [Simulator] Running ./vendor/bin/run toolkit:build-dist + > ./vendor/bin/gitleaks directory dist --no-banner --report-path=gitleaks-report.json + [Simulator] Running ./vendor/bin/gitleaks directory dist --no-banner --report-path=gitleaks-report.json + ------------------ --- + Findings Found 0 + Findings Ignored 0 + Findings Active 0 + ------------------ --- -- command: 'toolkit:run-gitleaks --tag=1.0.2 --os=linux_x32' +- command: 'toolkit:run-gitleaks --tag=1.0.2 --os=linux_x32 --sha256=deadbeef' + configuration: [] + resources: + - file: gitleaks-report.json + content: '[]' expectations: - contains: | - Downloading from https://github.com/gitleaks/gitleaks/releases/download/v1.0.2/gitleaks_1.0.2_linux_x32.tar.gz - [Simulator] Simulating Exec('./vendor/bin/gitleaks detect --no-banner -v --no-git --follow-symlinks') - [Simulator] Running ./vendor/bin/gitleaks detect --no-banner -v --no-git --follow-symlinks + Skip download, binary was found. + [Simulator] Simulating Exec('./vendor/bin/run toolkit:build-dist') + [Simulator] Running ./vendor/bin/run toolkit:build-dist + > ./vendor/bin/gitleaks detect --source=dist --no-banner -v --no-git --follow-symlinks --report-path=gitleaks-report.json + [Simulator] Running ./vendor/bin/gitleaks detect --source=dist --no-banner -v --no-git --follow-symlinks --report-path=gitleaks-report.json + ------------------ --- + Findings Found 0 + Findings Ignored 0 + Findings Active 0 + ------------------ --- + +- command: 'toolkit:run-gitleaks' + configuration: [] + resources: + - from: sample-gitleaks-report.json + to: gitleaks-report.json + - file: .toolkit-mock/0.0.103/api/v1/gitleaks-ignores.json + content: | + empty + expectations: + - contains: | + Skip download, binary was found. + [Simulator] Simulating Exec('./vendor/bin/run toolkit:build-dist') + [Simulator] Running ./vendor/bin/run toolkit:build-dist + > ./vendor/bin/gitleaks directory dist --no-banner -v --follow-symlinks --report-path=gitleaks-report.json + [Simulator] Running ./vendor/bin/gitleaks directory dist --no-banner -v --follow-symlinks --report-path=gitleaks-report.json + Findings: + ========= + Detected a Generic API Key, potentially exposing access to various services and sensitive operations. + ===================================================================================================== + File: web/core/assets/vendor/ckeditor5/find-and-replace/find-and-replace.js + Fingerprint: web/core/assets/vendor/ckeditor5/find-and-replace/find-and-replace.js:generic-api-key:5 + ToolkitFingerprint: 387a4077adcc1fc60f0135e79c0450ca91ca90b2a89c968b13bcb0771f04c29f + + Detected a Generic API Key, potentially exposing access to various services and sensitive operations. + ===================================================================================================== + File: web/core/assets/vendor/ckeditor5/special-characters/special-characters.js + Fingerprint: web/core/assets/vendor/ckeditor5/special-characters/special-characters.js:generic-api-key:5 + ToolkitFingerprint: f3b0bd000a796a123ef1f58368f5847bc7322bac8f2b2014b254eaa890a877b2 + + ------------------ --- + Findings Found 2 + Findings Ignored 0 + Findings Active 2 + ------------------ --- + +- command: 'toolkit:run-gitleaks' + configuration: [] + resources: + - from: sample-gitleaks-report.json + to: gitleaks-report.json + - file: .toolkit-mock/0.0.103/api/v1/gitleaks-ignores.json + content: | + {"gitleaks_ignores":[ + "f3b0bd000a796a123ef1f58368f5847bc7322bac8f2b2014b254eaa890a877b2" + ]} + - file: .leaksignore + content: | + web/core/assets/vendor/ckeditor5/find-and-replace/find-and-replace.js:generic-api-key:5 + expectations: + - contains: | + Skip download, binary was found. + [Simulator] Simulating Exec('./vendor/bin/run toolkit:build-dist') + [Simulator] Running ./vendor/bin/run toolkit:build-dist + > ./vendor/bin/gitleaks directory dist --no-banner -v --follow-symlinks --report-path=gitleaks-report.json + [Simulator] Running ./vendor/bin/gitleaks directory dist --no-banner -v --follow-symlinks --report-path=gitleaks-report.json + + ------------------ --- + Findings Found 2 + Findings Ignored 2 + Findings Active 0 + ------------------ --- diff --git a/tests/fixtures/commands/notifications.yml b/tests/fixtures/commands/notifications.yml index 08bf31c23..3a52501ee 100644 --- a/tests/fixtures/commands/notifications.yml +++ b/tests/fixtures/commands/notifications.yml @@ -11,7 +11,7 @@ - CI=false - QA_WEBSITE_URL=http://ec-behat.com resources: - - file: .toolkit-mock/0.0.85/api/v1/notifications.json + - file: .toolkit-mock/0.0.103/api/v1/notifications.json content: | [{ "title": "Notification title 1", @@ -28,7 +28,6 @@ Description of notification 1 - Notification title 2 ==================== @@ -44,7 +43,7 @@ - CI=false - QA_WEBSITE_URL=http://ec-behat.com resources: - - file: .toolkit-mock/0.0.85/api/v1/notifications.json + - file: .toolkit-mock/0.0.103/api/v1/notifications.json content: | [{ "title": "Notification title 1", diff --git a/tests/fixtures/commands/release.yml b/tests/fixtures/commands/release.yml index 8c6df1586..bd4c96b42 100644 --- a/tests/fixtures/commands/release.yml +++ b/tests/fixtures/commands/release.yml @@ -93,7 +93,6 @@ ->version('9.6.0') ->setBody('## Version 9.6.0 - ') - command: 'toolkit:changelog-write 9.6.0 --show-pr --show-name --full-link' diff --git a/tests/fixtures/commands/tests.yml b/tests/fixtures/commands/tests.yml index fc0cad477..7822df99c 100644 --- a/tests/fixtures/commands/tests.yml +++ b/tests/fixtures/commands/tests.yml @@ -142,7 +142,7 @@ [error] The following standards are missing, please add them to the configuration file 'phpcs.xml'. ./vendor/drupal/coder/coder_sniffer/Drupal ./vendor/drupal/coder/coder_sniffer/DrupalPractice - ./vendor/ec-europa/qa-automation/phpcs/QualityAssurance + ./vendor/ec-europa/toolkit/phpcs/QualityAssurance - command: 'toolkit:check-phpcs-requirements' configuration: [] diff --git a/tests/fixtures/samples/sample-composer.lock b/tests/fixtures/samples/sample-composer.lock index 5d784403e..610ae0341 100644 --- a/tests/fixtures/samples/sample-composer.lock +++ b/tests/fixtures/samples/sample-composer.lock @@ -3283,7 +3283,7 @@ }, { "name": "drupal/core", - "version": "10.5.1", + "version": "10.5.10", "source": { "type": "git", "url": "https://github.com/drupal/core.git", @@ -8416,88 +8416,6 @@ }, "time": "2022-09-16T12:52:18+00:00" }, - { - "name": "openeuropa/oe_dashboard_agent", - "version": "0.3.0", - "source": { - "type": "git", - "url": "https://github.com/openeuropa/oe_dashboard_agent.git", - "reference": "cb54ee2b0c5dc53c8e201a8bcd1e3fbf1c30e8dd" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/openeuropa/oe_dashboard_agent/zipball/cb54ee2b0c5dc53c8e201a8bcd1e3fbf1c30e8dd", - "reference": "cb54ee2b0c5dc53c8e201a8bcd1e3fbf1c30e8dd", - "shasum": "" - }, - "require": { - "drupal/core": "^8.9 || ^9.1", - "php": ">=7.3" - }, - "require-dev": { - "composer/installers": "~1.5", - "drupal/config_devel": "~1.2", - "drupal/core-composer-scaffold": "^8.9 || ^9.1", - "drupal/datetime_testing": "1.x-dev", - "drupal/drupal-extension": "^4.0", - "drush/drush": "^10.3", - "openeuropa/code-review": "^1.6", - "openeuropa/drupal-core-require-dev": "^8.9 || ^9.1", - "openeuropa/task-runner-drupal-project-symlink": "^1.0", - "phpspec/prophecy-phpunit": "^1 || ^2" - }, - "type": "drupal-module", - "extra": { - "composer-exit-on-patch-failure": true, - "enable-patching": true, - "installer-paths": { - "build/core": [ - "type:drupal-core" - ], - "build/profiles/contrib/{$name}": [ - "type:drupal-profile" - ], - "build/modules/contrib/{$name}": [ - "type:drupal-module" - ], - "build/themes/contrib/{$name}": [ - "type:drupal-theme" - ] - }, - "drupal-scaffold": { - "locations": { - "web-root": "./build" - } - } - }, - "autoload": { - "psr-4": { - "Drupal\\oe_dashboard_agent\\": "./src/" - } - }, - "autoload-dev": { - "psr-4": { - "Drupal\\Tests\\oe_dashboard_agent\\": "./tests/src/" - } - }, - "scripts": { - "post-install-cmd": [ - "./vendor/bin/run drupal:site-setup" - ], - "post-update-cmd": [ - "./vendor/bin/run drupal:site-setup" - ] - }, - "license": [ - "EUPL-1.2" - ], - "description": "OpenEuropa Dashboard Agent.", - "support": { - "source": "https://github.com/openeuropa/oe_dashboard_agent/tree/0.3.0", - "issues": "https://github.com/openeuropa/oe_dashboard_agent/issues" - }, - "time": "2021-08-16T08:11:50+00:00" - }, { "name": "pear/archive_tar", "version": "1.4.14", diff --git a/tests/fixtures/samples/sample-dump.sql.gz b/tests/fixtures/samples/sample-dump.sql.gz index e69de29bb..421376db9 100644 --- a/tests/fixtures/samples/sample-dump.sql.gz +++ b/tests/fixtures/samples/sample-dump.sql.gz @@ -0,0 +1 @@ +dummy diff --git a/tests/fixtures/samples/sample-gitleaks-report.json b/tests/fixtures/samples/sample-gitleaks-report.json new file mode 100644 index 000000000..e4db66184 --- /dev/null +++ b/tests/fixtures/samples/sample-gitleaks-report.json @@ -0,0 +1,42 @@ +[ + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 5, + "EndLine": 5, + "StartColumn": 14499, + "EndColumn": 14538, + "Match": "r.KeystrokeHandler,this._focusables=new ", + "Secret": "this._focusables=new", + "File": "web\/core\/assets\/vendor\/ckeditor5\/find-and-replace\/find-and-replace.js", + "SymlinkFile": "", + "Commit": "", + "Entropy": 3.9841838, + "Author": "", + "Email": "", + "Date": "", + "Message": "", + "Tags": [], + "Fingerprint": "web\/core\/assets\/vendor\/ckeditor5\/find-and-replace\/find-and-replace.js:generic-api-key:5" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 5, + "EndLine": 5, + "StartColumn": 12180, + "EndColumn": 12220, + "Match": "c.KeystrokeHandler,this._focusCycler=new ", + "Secret": "this._focusCycler=new", + "File": "web\/core\/assets\/vendor\/ckeditor5\/special-characters\/special-characters.js", + "SymlinkFile": "", + "Commit": "", + "Entropy": 4.106603, + "Author": "", + "Email": "", + "Date": "", + "Message": "", + "Tags": [], + "Fingerprint": "web\/core\/assets\/vendor\/ckeditor5\/special-characters\/special-characters.js:generic-api-key:5" + } +] diff --git a/tests/fixtures/samples/sample-opts.yml b/tests/fixtures/samples/sample-opts.yml index 3bf8f61fe..6eda0bc00 100644 --- a/tests/fixtures/samples/sample-opts.yml +++ b/tests/fixtures/samples/sample-opts.yml @@ -13,4 +13,4 @@ upgrade_commands: - ./vendor/bin/drush cache:rebuild ephemeral extra_pkgs: - php8.1-zip -php_version: '8.1' +php_version: '8.3' diff --git a/tests/fixtures/samples/sample-phpcs.xml b/tests/fixtures/samples/sample-phpcs.xml index 9d66bf673..c8585d7e2 100644 --- a/tests/fixtures/samples/sample-phpcs.xml +++ b/tests/fixtures/samples/sample-phpcs.xml @@ -5,7 +5,7 @@ - +