From 1f3287820cbeaaab469cfc181eb54071cecc8d0f Mon Sep 17 00:00:00 2001 From: Civics Team Date: Thu, 2 May 2024 11:44:30 -0700 Subject: [PATCH] Make FeedInactiveDate optional in the Metadata spec. This should not be set for evergreen feeds. PiperOrigin-RevId: 630132146 --- base_spec.xsd | 141 +++++++++++++++++++++++++++++++++++++++++ civics_cdf_spec.xsd | 47 +------------- metadata_spec.xsd | 125 +++++++++++++++++++++++++++++++++++++ rules.py | 36 +++++++---- tests/rules_test.py | 39 +++++++++--- tests/samples_test.py | 142 ------------------------------------------ validator.py | 2 + version.py | 2 +- 8 files changed, 324 insertions(+), 210 deletions(-) create mode 100644 base_spec.xsd create mode 100644 metadata_spec.xsd delete mode 100644 tests/samples_test.py diff --git a/base_spec.xsd b/base_spec.xsd new file mode 100644 index 0000000..ddbeed5 --- /dev/null +++ b/base_spec.xsd @@ -0,0 +1,141 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/civics_cdf_spec.xsd b/civics_cdf_spec.xsd index a2e1a8d..61307d7 100644 --- a/civics_cdf_spec.xsd +++ b/civics_cdf_spec.xsd @@ -1,26 +1,8 @@ + - - - - - - - - - - - - - - - - - - - @@ -70,15 +52,6 @@ - - - - - - - - - @@ -106,24 +79,6 @@ - - - - - - - - - - - - - - - - - - diff --git a/metadata_spec.xsd b/metadata_spec.xsd new file mode 100644 index 0000000..3785446 --- /dev/null +++ b/metadata_spec.xsd @@ -0,0 +1,125 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rules.py b/rules.py index 1c5b8c0..d2c4230 100755 --- a/rules.py +++ b/rules.py @@ -1423,7 +1423,7 @@ def check_specific(self, parties): class IndependentPartyName(base.BaseRule): - """Warns on parties that contain common names indicating they are an indpendent party. + """Warns on parties that contain common names indicating they are an independent party. These should instead supply the IsIndependent attribute. """ @@ -1925,7 +1925,7 @@ def check(self, element): raise loggers.ElectionError.from_message("Missing URI value.", [element]) parsed_url = urlparse(url) - discrepencies = [] + discrepancies = [] social_media_platform = ["facebook", "twitter", "wikipedia", "instagram", "youtube", "website", "linkedin", "line", "ballotpedia", "tiktok"] @@ -1933,16 +1933,16 @@ def check(self, element): try: url.encode("ascii") except UnicodeEncodeError: - discrepencies.append("not ascii encoded") + discrepancies.append("not ascii encoded") if parsed_url.scheme not in {"http", "https"}: - discrepencies.append("protocol - invalid") + discrepancies.append("protocol - invalid") if not parsed_url.netloc: - discrepencies.append("domain - missing") - if discrepencies: + discrepancies.append("domain - missing") + if discrepancies: msg = ( "The provided URI, {}, is invalid for the following reasons: {}." - .format(url.encode("ascii", "ignore"), ", ".join(discrepencies)) + .format(url.encode("ascii", "ignore"), ", ".join(discrepancies)) ) raise loggers.ElectionError.from_message(msg, [element]) @@ -2024,11 +2024,14 @@ def elements(self): def check(self, element): url = element.text.strip() parsed_url = urlparse(url) - if "youtube" in parsed_url.netloc and (parsed_url.path in ["", "/"] - or "watch" in parsed_url.path - or "playlist" in parsed_url.path): + if "youtube" in parsed_url.netloc and ( + parsed_url.path in ["", "/"] + or "watch" in parsed_url.path + or "playlist" in parsed_url.path + or "hashtag" in parsed_url.path + ): raise loggers.ElectionError.from_message( - "'{}' is not a expected value for a youtube channel.".format(url), + "'{}' is not an expected value for a youtube channel.".format(url), [element]) @@ -3307,7 +3310,7 @@ def check(self, election_report_element): class MultipleInternationalizedTextWithSameLanguageCode(base.BaseRule): - """Checks for muliple InternationalizedText with the same language code.""" + """Checks for multiple InternationalizedText with the same language code.""" def elements(self): return _INTERNATIONALIZED_TEXT_ELEMENTS @@ -3697,6 +3700,7 @@ class RuleSet(enum.Enum): COMMITTEE = 3 ELECTION_DATES = 4 ELECTION_RESULTS = 5 + METADATA = 6 # To add new rules, create a new class, inherit the base rule, @@ -3822,6 +3826,13 @@ class RuleSet(enum.Enum): COMMON_RULES + ELECTION_RULES + (UnreferencedEntitiesElectionDates,) ) +METADATA_RULES = ( + Schema, + Encoding, + OptionalAndEmpty, + UniqueLabel, +) + ALL_RULES = frozenset( COMMON_RULES + ELECTION_RULES @@ -3829,4 +3840,5 @@ class RuleSet(enum.Enum): + OFFICEHOLDER_RULES + COMMITTEE_RULES + ELECTION_DATES_RULES + + METADATA_RULES ) diff --git a/tests/rules_test.py b/tests/rules_test.py index bee62eb..cd94dfa 100644 --- a/tests/rules_test.py +++ b/tests/rules_test.py @@ -5509,9 +5509,11 @@ def testYTWatchUrlReturnError(self): """ with self.assertRaises(loggers.ElectionError) as cm: self.valid_yt_url.check(etree.fromstring(root_string)) - self.assertEqual(cm.exception.log_entry[0].message, - ("'https://www.youtube.com/watch?v=k-F_qYKkqaVxbA' is not " - "a expected value for a youtube channel.")) + self.assertEqual( + cm.exception.log_entry[0].message, + "'https://www.youtube.com/watch?v=k-F_qYKkqaVxbA' is not an expected" + " value for a youtube channel.", + ) self.assertEqual(cm.exception.log_entry[0].elements[0].tag, "Uri") def testYTPlaylistUrlReturnError(self): @@ -5522,9 +5524,26 @@ def testYTPlaylistUrlReturnError(self): """ with self.assertRaises(loggers.ElectionError) as cm: self.valid_yt_url.check(etree.fromstring(root_string)) - self.assertEqual(cm.exception.log_entry[0].message, ( - "'https://www.youtube.com/playlist?list=PLCvVBOK6lIHsfkBVt0oCFMSRz_grSwC4N' is not " - "a expected value for a youtube channel.")) + self.assertEqual( + cm.exception.log_entry[0].message, + "'https://www.youtube.com/playlist?list=PLCvVBOK6lIHsfkBVt0oCFMSRz_grSwC4N'" + " is not an expected value for a youtube channel.", + ) + self.assertEqual(cm.exception.log_entry[0].elements[0].tag, "Uri") + + def testYTHashtagUrlReturnError(self): + root_string = """ + + + + """ + with self.assertRaises(loggers.ElectionError) as cm: + self.valid_yt_url.check(etree.fromstring(root_string)) + self.assertEqual( + cm.exception.log_entry[0].message, + "'https://www.youtube.com/hashtag/xyz' is not an expected value for a" + " youtube channel.", + ) self.assertEqual(cm.exception.log_entry[0].elements[0].tag, "Uri") def testBasicYTUrlReturnError(self): @@ -5535,9 +5554,11 @@ def testBasicYTUrlReturnError(self): """ with self.assertRaises(loggers.ElectionError) as cm: self.valid_yt_url.check(etree.fromstring(root_string)) - self.assertEqual(cm.exception.log_entry[0].message, - ("'https://www.youtube.com/' is not a expected value for " - "a youtube channel.")) + self.assertEqual( + cm.exception.log_entry[0].message, + "'https://www.youtube.com/' is not an expected value for a youtube" + " channel.", + ) self.assertEqual(cm.exception.log_entry[0].elements[0].tag, "Uri") diff --git a/tests/samples_test.py b/tests/samples_test.py deleted file mode 100644 index cafbcd7..0000000 --- a/tests/samples_test.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Tests for google3.third_party.py.civics_cdf_validator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from absl import flags -from absl.testing import absltest -from civics_cdf_validator import base -from civics_cdf_validator import gpunit_rules -from civics_cdf_validator import loggers -from civics_cdf_validator import rules -import pytest - - -FLAGS = flags.FLAGS - - -@pytest.mark.skip(reason='skip samples test during local development') -class SamplesTest(absltest.TestCase): - """Test that all sample files pass validation.""" - - ocd_ids = [ - 'ocd-division/country:us', - 'ocd-division/country:us/state:tx', - 'ocd-division/country:us/state:vt', - 'ocd-division/country:us/state:va', - 'ocd-division/country:us/state:ca', - 'ocd-division/country:us/state:fl', - 'ocd-division/country:us/state:va/county:albemarle', - ] - - def setUp(self): - super(SamplesTest, self).setUp() - # OCD-ID rules don't work inside google3. - ocd_id_rules = {rules.ElectoralDistrictOcdId, rules.GpUnitOcdId} - # Election dates check for dates to not be in the past. Since our sample - # files will always, at some point, end up being in the past we are - # removing date rules from these tests. - date_rules = {rules.ElectionStartDates, rules.ElectionEndDatesInThePast, - rules.ElectionEndDatesOccurAfterStartDates} - self.election_rules = ( - set(rules.ELECTION_RULES) - ocd_id_rules - date_rules) - self.officeholder_rules = ( - set(rules.OFFICEHOLDER_RULES) - ocd_id_rules) - - def testOfficeholderSampleFeed(self): - self._TestFile('officeholder_sample_feed.xml', self.officeholder_rules) - - def testPostElectionSampleFeedPrecincts(self): - self._TestFile( - 'post_election_sample_feed_precincts.xml', - self.election_rules, - expected_errors=19, - expected_warnings=14, - ) - - def testPostRetentionContestSampleFeedSummary(self): - self._TestFile( - 'post_retention_contest_sample_feed_summary.xml', - self.election_rules, - ) - - def testPreRetentionContestSampleFeed(self): - self._TestFile( - 'pre_retention_contest_sample_feed.xml', - self.election_rules, - ) - - def testPostRetentionContestSampleFeedPrecincts(self): - self._TestFile( - 'post_retention_contest_sample_feed_precincts.xml', - self.election_rules, - expected_errors=19, - ) - - def testPostElectionSampleFeedSummary(self): - self._TestFile( - 'post_election_sample_feed_summary.xml', - self.election_rules, - expected_warnings=14, - ) - - def testPreElectionSampleFeed(self): - self._TestFile( - 'pre_election_sample_feed.xml', - self.election_rules, - expected_warnings=1) - - def testBallotMeasureContestSampleFeed(self): - self._TestFile('ballot_measure_contest_sample_feed.xml', - self.election_rules) - - def testBallotMeasureContestWithResultsSampleFeed(self): - self._TestFile('ballot_measure_contest_sample_feed_with_results.xml', - self.election_rules) - - def testMultiElectionSampleFeed(self): - self._TestFile('multi_election_sample_feed.xml', self.election_rules) - - def _TestFile( - self, - filename, - rules_to_check, - expected_errors=0, - expected_warnings=0, - ): - sample_file = os.path.join( - FLAGS.test_srcdir, - 'google3/third_party/py/civics_cdf_validator/' - 'samples/' + filename) - schema_file = os.path.join( - FLAGS.test_srcdir, - 'google3/third_party/py/civics_cdf_validator/' - 'civics_cdf_spec.xsd') - gpunit_ocdid_validator = gpunit_rules.GpUnitOcdIdValidator( - 'us', None, False, self.ocd_ids - ) - registry = base.RulesRegistry( - election_file=sample_file, - schema_file=schema_file, - rule_options={}, - rule_classes_to_check=rules_to_check, - ocd_id_validator=gpunit_ocdid_validator, - ) - - registry.check_rules() - registry.print_exceptions(0, True) - self.assertEqual( - expected_errors, - registry.exceptions_wrapper.count_logs_with_exception_type( - loggers.ElectionError)) - self.assertEqual( - expected_warnings, - registry.exceptions_wrapper.count_logs_with_exception_type( - loggers.ElectionWarning)) - - -if __name__ == '__main__': - absltest.main() diff --git a/validator.py b/validator.py index 8142198..b747219 100755 --- a/validator.py +++ b/validator.py @@ -255,6 +255,8 @@ def filter_all_rules_using_user_arg(rules_allowlist, rule_set, rules_blocklist): rule_names = [x.__name__ for x in rules.ELECTION_DATES_RULES] elif rule_set == rules.RuleSet.ELECTION_RESULTS: rule_names = [x.__name__ for x in rules.ELECTION_RESULTS_RULES] + elif rule_set == rules.RuleSet.METADATA: + rule_names = [x.__name__ for x in rules.METADATA_RULES] else: raise AssertionError("Invalid rule_set: " + rule_set) if rules_blocklist: diff --git a/version.py b/version.py index e07deb9..ae558aa 100644 --- a/version.py +++ b/version.py @@ -5,4 +5,4 @@ See https://packaging.python.org/guides/single-sourcing-package-version/ """ -__version__ = '1.41.dev2' +__version__ = '1.42.dev1'