diff --git a/bugbot/rules/web_platform_features.py b/bugbot/rules/web_platform_features.py index 36de880c2..41e3bed7f 100644 --- a/bugbot/rules/web_platform_features.py +++ b/bugbot/rules/web_platform_features.py @@ -13,17 +13,21 @@ Any, Generic, Iterable, + Literal, Mapping, MutableMapping, Optional, Sequence, TypeVar, + cast, ) from urllib import parse from google.cloud import bigquery +from libmozdata.bugzilla import Bugzilla +from requests.exceptions import HTTPError -from bugbot import gcp +from bugbot import gcp, logger, spec_mapping, utils from bugbot.bzcleaner import Bug, BzCleaner Json = None | str | int | float | Sequence["Json"] | Mapping[str, "Json"] @@ -87,6 +91,43 @@ def __bool__(self) -> bool: return bool(self.add or self.remove) +@dataclass +class BugzillaNewBug: + """Representation of a new bug to be created""" + + summary: str + product: str + component: str + description: str + type: Literal["defect"] | Literal["enhancement"] | Literal["task"] + version: str = "unspecified" + keywords: Optional[list[str]] = None + whiteboard: Optional[str] = None + see_also: Optional[list[str]] = None + user_story: Optional[str] = None + url: Optional[str] = None + + def to_json(self) -> Mapping[str, Json]: + rv: dict[str, Json] = { + "summary": self.summary, + "product": self.product, + "component": self.component, + "description": self.description, + "version": self.version, + "type": self.type, + } + for value, name in [ + (self.whiteboard, "status_whiteboard"), + (self.keywords, "keywords"), + (self.see_also, "see_also"), + (self.user_story, "cf_user_story"), + (self.url, "url"), + ]: + if value is not None: + rv[name] = value + return rv + + @dataclass class BugzillaUpdate: """Representation of bug changes for use with the Bugzilla ReST API""" @@ -287,11 +328,36 @@ class FeatureData: supported_browsers: set[str] sp_issue: Optional[int] spec_url: set[str] + name: Optional[str] = None + description: Optional[str] = None def is_supported(self) -> bool: return {"firefox", "firefox_android"}.issubset(self.supported_browsers) +def feature_keywords(feature: FeatureData) -> set[str]: + rv = set() + if {"chrome", "chrome_android"}.issubset(feature.supported_browsers): + rv.add("parity-chrome") + if {"safari", "safari_ios"}.issubset(feature.supported_browsers): + rv.add("parity-safari") + return rv + + +def feature_links(feature: FeatureData) -> set[str]: + links = set( + [ + f"https://web-platform-dx.github.io/web-features-explorer/features/{feature.feature}/" + ] + ) + if feature.sp_issue is not None: + links.add( + f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}" + ) + links |= feature.spec_url + return links + + @dataclass class FeatureBug: """Bug that represents a web-feature""" @@ -313,10 +379,7 @@ def expected_keywords(self) -> set[str]: rv.add("web-feature") if not self.is_supported(): for feature in self.features.values(): - if {"chrome", "chrome_android"}.issubset(feature.supported_browsers): - rv.add("parity-chrome") - if {"safari", "safari_ios"}.issubset(feature.supported_browsers): - rv.add("parity-safari") + rv |= feature_keywords(feature) return rv def missing_keywords(self) -> set[str]: @@ -324,15 +387,8 @@ def missing_keywords(self) -> set[str]: def expected_links(self) -> set[str]: links = set() - for feature_name, feature in self.features.items(): - links.add( - f"https://web-platform-dx.github.io/web-features-explorer/features/{feature_name}/" - ) - if feature.sp_issue is not None: - links.add( - f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}" - ) - links |= feature.spec_url + for feature in self.features.values(): + links |= feature_links(feature) return links def missing_links(self) -> set[str]: @@ -366,6 +422,79 @@ def remove_links(self) -> set[str]: _DataType = TypeVar("_DataType") +class CreateRule(ABC, Generic[_DataType]): + """Rule for creating new bugs based on BigQuery data""" + + def __init__(self, client: bigquery.Client): + self.client = client + + @abstractmethod + def get_data(self) -> _DataType: + ... + + @abstractmethod + def create(self, data: _DataType) -> Mapping[str, BugzillaNewBug]: + ... + + def run(self) -> Mapping[str, BugzillaNewBug]: + data: _DataType = self.get_data() + return self.create(data) + + +class FirefoxOnlyMissing(CreateRule): + def get_data(self) -> list[FeatureData]: + query = """ +SELECT + features.feature, + features.name, + features.description, + (SELECT ARRAY_AGG(browser) FROM UNNEST(features.support)) AS supported_browsers, + features.spec as spec_url, + sp_mozilla.issue as sp_issue +FROM `web_features.features_latest` AS features +LEFT JOIN `webcompat_knowledge_base.bugzilla_bugs` AS bugs ON + features.feature IN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature")) +LEFT JOIN `standards_positions.mozilla_standards_positions` AS sp_mozilla ON + sp_mozilla.web_feature = features.feature +WHERE + "safari" in UNNEST(features.support.browser) AND + "chrome" IN UNNEST(features.support.browser) AND + "firefox" NOT IN UNNEST(features.support.browser) AND + bugs.number IS NULL +""" + return [ + FeatureData( + feature=row.feature, + spec_url=set(row.spec_url), + supported_browsers=set(row.supported_browsers), + sp_issue=row.sp_issue, + name=row.name, + description=row.description, + ) + for row in self.client.query(query) + ] + + def create(self, data: list[FeatureData]) -> Mapping[str, BugzillaNewBug]: + rv = {} + spec_mapper = spec_mapping.SpecMapper.load() + for feature in data: + product, component = spec_mapper.map_urls(feature.spec_url) + spec_url = feature.spec_url.pop() + rv[feature.feature] = BugzillaNewBug( + summary=f"[meta] Implement {feature.name}", + product=product, + component=component, + description=f"Implement {feature.name}:\n{feature.description}", + type="enhancement", + keywords=["web-feature"] + list(feature_keywords(feature)), + user_story=f"web-feature: {feature.feature}", + url=spec_url, + see_also=[item for item in feature_links(feature) if item != spec_url], + ) + + return rv + + class UpdateRule(ABC, Generic[_DataType]): """Rule for updating bugs based on BigQuery data""" @@ -598,6 +727,8 @@ def update( class WebPlatformFeatures(BzCleaner): def __init__(self) -> None: super().__init__() + self.create_bugs: dict[str, BugzillaNewBug] = {} + self.bugs_created: dict[int, BugzillaNewBug] = {} self.bug_updates: dict[int, FeatureBugUpdate] = defaultdict(FeatureBugUpdate) def description(self) -> str: @@ -610,24 +741,106 @@ def has_default_products(self) -> bool: return False def columns(self) -> list[str]: - return ["id", "summary", "changes", "whiteboard", "user_story"] + return ["id", "summary", "change_type", "changes", "whiteboard", "user_story"] + + def get_bugs( + self, + date: str = "today", + bug_ids: list[int] = [], + chunk_size: Optional[int] = None, + ) -> dict[str, Mapping[str, Any]]: + bugs = super().get_bugs(date, bug_ids, chunk_size) + + if self.create_bugs: + bugs_for_features = self.get_feature_bugs(set(self.create_bugs.keys())) + else: + bugs_for_features = {} + + for i, (feature, bug) in enumerate(self.create_bugs.items()): + if feature in bugs_for_features: + # A bug was already created for this feature + continue + + if self.dryrun or self.test_mode: + response = {"id": i} + logger.info( + f"A bug '{bug.summary}` would be created with:\n{bug.to_json()}", + ) + else: + try: + response = utils.create_bug(cast(dict, bug.to_json())) + except HTTPError: + logger.error( + f"Failed to create bug '{bug.summary}'", + ) + continue + + bug_id = response["id"] + assert isinstance(bug_id, int) + self.bugs_created[bug_id] = bug + bugs[str(bug_id)] = { + "id": bug_id, + "summary": bug.summary, + "url": bug.url, + "see_also": bug.component, + "keywords": bug.keywords, + "whiteboard": bug.whiteboard, + "cf_user_story": bug.user_story, + "status": "NEW", + "resolution": "", + "change_type": "create", + "changes": bug, + "user_story": bug.user_story, + } + + return bugs + + def get_feature_bugs(self, features: set[str]) -> Mapping[str, int]: + """Get the list of existing bugs for specific features""" + data: dict[str, int] = {} + + def handler(bug: Mapping[str, Any], data: dict[str, int]) -> None: + for _, key, value in parse_user_story(bug["cf_user_story"]): + if key == "web-feature": + if value in features: + data[value] = bug["id"] + + Bugzilla( + { + "keywords": "web-feature", + "keywords_type": "allwords", + "f1": "cf_user_story", + "o1": "substring", + "v1": "web-feature", + "f2": "cf_user_story", + "o2": "anywordssubstr", + "v2": ",".join(features), + }, + bugdata=data, + bughandler=handler, + ).wait() + + return data def handle_bug(self, bug: Bug, data: dict[str, Any]) -> Optional[Bug]: bug_id_str = str(bug["id"]) bug_id_int = int(bug["id"]) - if bug_id_int not in self.bug_updates: - return None - bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug) + if bug_id_int in self.bug_updates: + bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug) + if not bugzilla_update: + return None - if bugzilla_update: self.autofix_changes[bug_id_str] = bugzilla_update.to_json() data[bug_id_str] = { + "change_type": "update", "changes": bugzilla_update, "whiteboard": bug["whiteboard"], "user_story": bug["cf_user_story"], } return bug + elif bug_id_int in self.bugs_created: + return bug return None @@ -648,6 +861,10 @@ def get_bz_params(self, date: str) -> dict[str, str | int | list[str] | list[int def get_bug_updates(self) -> None: project = "moz-fx-dev-dschubert-wckb" client = gcp.get_bigquery_client(project, ["cloud-platform", "drive"]) + + for create_rule in [FirefoxOnlyMissing(client)]: + self.create_bugs.update(create_rule.run()) + for update_rule in [ FeatureRenames(client), InvalidFeatures(client), diff --git a/bugbot/spec_mapping.py b/bugbot/spec_mapping.py new file mode 100644 index 000000000..65a9c1a6c --- /dev/null +++ b/bugbot/spec_mapping.py @@ -0,0 +1,155 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# mypy: disallow-untyped-defs +"""Map web specification URLs to Bugzilla (product, component) pairs.""" + +import itertools +import os +import re +from abc import ABC, abstractmethod +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from typing import Self +from urllib.parse import SplitResult, urlsplit + +import yaml + +ProductComponent = tuple[str, str] + + +@dataclass +class RuleMatch: + """A resolved (product, component)""" + + position: tuple[int, int] + product: str + component: str + + def __gt__(self, other: "RuleMatch") -> bool: + return (self.position[0], -self.position[1]) > ( + other.position[0], + -other.position[1], + ) + + +_order = itertools.count() + + +class Rule(ABC): + level: int = 0 + + def __init__( + self, + component: ProductComponent | None = None, + paths: Sequence["Rule"] = (), + ): + self.component = component + self.order = next(_order) + + @abstractmethod + def get(self, parsed: SplitResult) -> RuleMatch | None: + ... + + +class HostRule(Rule): + level = 0 + + def __init__( + self, + component: ProductComponent | None, + paths: Sequence["PathRule"] = (), + ): + super().__init__(component) + self.paths = paths + + def get(self, parsed: SplitResult) -> RuleMatch | None: + for path_rule in self.paths: + m = path_rule.get(parsed) + if m is not None: + return m + if self.component is not None: + return RuleMatch((self.level, self.order), *self.component) + return None + + +class PathRule(Rule): + level = 1 + + def __init__( + self, + path: str | None, + component: ProductComponent, + ): + self.path = re.compile(path) if path is not None else None + super().__init__(component) + + def get(self, parsed: SplitResult) -> RuleMatch | None: + if self.path is None or self.path.search(parsed.path) is not None: + assert self.component is not None + return RuleMatch((self.level, self.order), *self.component) + return None + + +def parse_component(value: str) -> ProductComponent: + if " :: " not in value: + raise ValueError(f"Invalid component '{value}'") + product, _, component = value.partition(" :: ") + return product, component + + +class SpecMapper: + def __init__(self, rules: dict[str, HostRule], default: ProductComponent): + self.rules = rules + self.default = default + + @classmethod + def load( + cls, path: str = os.path.join(os.path.dirname(__file__), "spec_mapping.yml") + ) -> Self: + with open(path) as f: + data = yaml.safe_load(f) + rules: dict[str, HostRule] = {} + default: ProductComponent | None = None + for entry in data: + component = entry.get("component") + if "host" not in entry: + if default is not None: + raise ValueError( + f"Got multiple default components second in entry {entry}" + ) + default = parse_component(component) + continue + paths = [ + PathRule(path_entry["path"], parse_component(path_entry["component"])) + for path_entry in entry.get("paths", []) + ] + rules[entry["host"]] = HostRule( + parse_component(component) if component is not None else None, + paths, + ) + if default is None: + raise ValueError("Missing default component") + return cls(rules, default) + + def map_url(self, url: str) -> RuleMatch | None: + """Return the most specific :class:`Match` for ``url``, or ``None``.""" + parsed = urlsplit(url) + host_rule = self.rules.get(parsed.hostname or "") + if host_rule is None: + return None + return host_rule.get(parsed) + + def map_urls(self, urls: Iterable[str]) -> ProductComponent: + """Pick a plausible (product, component) for a feature with given spec URLs.""" + best_match: RuleMatch | None = None + for url in urls: + rule_match = self.map_url(url) + if rule_match is None: + continue + if best_match is None or rule_match > best_match: + best_match = rule_match + if best_match is None: + return self.default + return best_match.product, best_match.component diff --git a/bugbot/spec_mapping.yml b/bugbot/spec_mapping.yml new file mode 100644 index 000000000..816bec910 --- /dev/null +++ b/bugbot/spec_mapping.yml @@ -0,0 +1,573 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# Maps web specification hosts/paths to Bugzilla "Product :: Component". +# Each entry matches a URL host; `paths` sub-rules match the URL path +# (as a regular expression) and take precedence over the host `component`. + +- component: 'Core :: General' +- host: drafts.csswg.org + component: 'Core :: Layout: General' + paths: + - path: /css-values- + component: 'Core :: CSS Parsing and Computation' + - path: /css-cascade- + component: 'Core :: CSS Parsing and Computation' + - path: /css-color-(\d|adjust|hdr|6) + component: 'Core :: CSS Parsing and Computation' + - path: /css-conditional- + component: 'Core :: CSS Parsing and Computation' + - path: /css-view-transitions- + component: 'Core :: CSS Parsing and Computation' + - path: /css-highlight-api- + component: 'Core :: CSS Parsing and Computation' + - path: /mediaqueries- + component: 'Core :: CSS Parsing and Computation' + - path: /css-viewport + component: 'Core :: CSS Parsing and Computation' + - path: /css-anchor-position- + component: 'Core :: CSS Parsing and Computation' + - path: /css-position- + component: 'Core :: CSS Parsing and Computation' + - path: /css-sizing- + component: 'Core :: CSS Parsing and Computation' + - path: /css-display- + component: 'Core :: CSS Parsing and Computation' + - path: /css-contain- + component: 'Core :: CSS Parsing and Computation' + - path: /css-variables- + component: 'Core :: CSS Parsing and Computation' + - path: /css-mixins- + component: 'Core :: CSS Parsing and Computation' + - path: /css-nesting- + component: 'Core :: CSS Parsing and Computation' + - path: /css-namespaces- + component: 'Core :: CSS Parsing and Computation' + - path: /selectors- + component: 'Core :: CSS Parsing and Computation' + - path: /css-pseudo- + component: 'Core :: CSS Parsing and Computation' + - path: /css-inline- + component: 'Core :: Layout: Block and Inline' + - path: /css-break- + component: 'Core :: Layout: Block and Inline' + - path: /css-text- + component: 'Core :: Layout: Text and Fonts' + - path: /css-fonts- + component: 'Core :: Layout: Text and Fonts' + - path: /css-text-decor- + component: 'Core :: Layout: Text and Fonts' + - path: /css-writing-modes- + component: 'Core :: Layout: Text and Fonts' + - path: /css-size-adjust- + component: 'Core :: Layout: Text and Fonts' + - path: /css-rhythm- + component: 'Core :: Layout: Text and Fonts' + - path: /css-counter-styles + component: 'Core :: Layout: Text and Fonts' + - path: /css-tables- + component: 'Core :: Layout: Tables' + - path: /css-grid- + component: 'Core :: Layout: Grid' + - path: /css-flexbox- + component: 'Core :: Layout' + - path: /css-multicol + component: 'Core :: Layout' + - path: /css-ruby- + component: 'Core :: Layout: Ruby' + - path: /css-scroll- + component: 'Core :: Layout: Scrolling and Overflow' + - path: /css-overflow- + component: 'Core :: Layout: Scrolling and Overflow' + - path: /css-overscroll + component: 'Core :: Layout: Scrolling and Overflow' + - path: /scroll-animations + component: 'Core :: DOM: Animation' + - path: /css-images- + component: 'Core :: Layout: Images, Video, and HTML Frames' + - path: /css-shapes- + component: 'Core :: Layout' + - path: /css-masking- + component: 'Core :: SVG' + - path: /filter-effects- + component: 'Core :: SVG' + - path: /compositing- + component: 'Core :: Web Painting' + - path: /geometry- + component: 'Core :: DOM: CSS Object Model' + - path: /css-page- + component: 'Core :: Printing: Output' + - path: /css-animations- + component: 'Core :: CSS Transitions and Animations' + - path: /css-transitions- + component: 'Core :: CSS Transitions and Animations' + - path: /css-easing- + component: 'Core :: CSS Transitions and Animations' + - path: /css-transforms- + component: 'Core :: CSS Transitions and Animations' + - path: /web-animations- + component: 'Core :: DOM: Animation' + - path: /motion- + component: 'Core :: DOM: Animation' + - path: /css-will-change + component: 'Core :: Web Painting' + - path: /css-shadow + component: 'Core :: Web Painting' + - path: /cssom-view + component: 'Core :: Layout' + - path: /cssom + component: 'Core :: DOM: CSS Object Model' + - path: /resize-observer + component: 'Core :: Layout' + - path: /css-gaps- + component: 'Core :: Layout' + - path: /css-box- + component: 'Core :: Layout' + - path: /css-align- + component: 'Core :: Layout' + - path: /css-logical- + component: 'Core :: Layout' + - path: /css-env- + component: 'Core :: Layout' + - path: /css-forms- + component: 'Core :: Layout' + - path: /css-ui- + component: 'Core :: Layout' + - path: /css-speech- + component: 'Core :: Disability Access APIs' + - path: /mathml + component: 'Core :: MathML' +- host: w3c.github.io + component: 'Core :: DOM: Core & HTML' + paths: + - path: /manifest + component: 'Firefox :: General' + - path: /badging + component: 'Firefox for Android :: PWA' + - path: /web-share-target + component: 'Core :: DOM: Web Share' + - path: /web-share + component: 'Core :: DOM: Web Share' + - path: /(accelerometer|ambient-light|gyroscope|magnetometer|orientation-sensor|sensors|compute-pressure|deviceorientation|device-posture|battery) + component: 'Core :: DOM: Device Interfaces' + - path: /aria + component: 'Core :: Disability Access APIs' + - path: /clipboard-apis + component: 'Core :: DOM: Copy & Paste and Drag & Drop' + - path: /editing + component: 'Core :: DOM: Editor' + - path: /edit-context + component: 'Core :: DOM: Editor' + - path: /webappsec-permissions-policy + component: 'Core :: DOM: Core & HTML' + - path: /webappsec-credential-management + component: 'Core :: DOM: Credential Management' + - path: /webappsec-referrer-policy + component: 'Core :: Networking' + - path: /webappsec- + component: 'Core :: DOM: Security' + - path: /trusted-types + component: 'Core :: DOM: Security' + - path: /webcrypto + component: 'Core :: DOM: Security' + - path: /(hr-time|performance-timeline|user-timing|server-timing|navigation-timing|resource-timing|element-timing|event-timing|paint-timing|largest-contentful-paint|long-animation-frames) + component: 'Core :: DOM: Performance APIs' + - path: /(requestidlecallback|longtasks) + component: 'Core :: DOM: Core & HTML' + - path: /IntersectionObserver + component: 'Core :: Layout' + - path: /IndexedDB + component: 'Core :: Storage: IndexedDB' + - path: /FileAPI + component: 'Core :: DOM: File' + - path: /web-locks + component: 'Core :: DOM: Core & HTML' + - path: /mediacapture-transform + component: 'Core :: WebRTC: Audio/Video' + - path: /mediacapture- + component: 'Core :: WebRTC: Audio/Video' + - path: /webrtc + component: 'Core :: WebRTC' + - path: /webtransport + component: 'Core :: Networking' + - path: /audio-session + component: 'Core :: Audio/Video: Playback' + - path: /encrypted-media + component: 'Core :: Audio/Video: Playback' + - path: /media-(capabilities|playback-quality|source) + component: 'Core :: Audio/Video: Playback' + - path: /mediasession + component: 'Core :: Audio/Video: Playback' + - path: /webcodecs + component: 'Core :: Audio/Video: Web Codecs' + - path: /remote-playback + component: 'Core :: Audio/Video' + - path: /webvtt + component: 'Core :: Audio/Video: Playback' + - path: /html-media-capture + component: 'Core :: Audio/Video: Recording' + - path: /picture-in-picture + component: 'Core :: DOM: Core & HTML' + - path: /mathml + component: 'Core :: MathML' + - path: /svgwg + component: 'Core :: SVG' + - path: /ServiceWorker + component: 'Core :: DOM: Service Workers' + - path: /push-api + component: 'Core :: DOM: Push Subscriptions' + - path: /gamepad + component: 'Core :: DOM: Device Interfaces' + - path: /geolocation + component: 'Core :: DOM: Geolocation' + - path: /(pointerevents|pointerlock|touch-events|uievents) + component: 'Core :: DOM: UI Events & Focus Handling' + - path: /vibration + component: 'Core :: DOM: Device Interfaces' + - path: /screen-orientation + component: 'Core :: DOM: Device Interfaces' + - path: /screen-wake-lock + component: 'Core :: DOM: Device Interfaces' + - path: /virtual-keyboard + component: 'Core :: Layout' + - path: /window-management + component: 'Core :: DOM: Core & HTML' + - path: /(payment-request|web-based-payment-handler|secure-payment-confirmation) + component: 'Core :: DOM: Web Payments' + - path: /webauthn + component: 'Core :: DOM: Web Authentication' + - path: /selection-api + component: 'Core :: DOM: Core & HTML' + - path: /DOM-Parsing + component: 'Core :: DOM: Serializers' + - path: /reporting + component: 'Core :: DOM: Core & HTML' + - path: /presentation-api + component: 'Core :: DOM: Core & HTML' + - path: /webdriver-bidi + component: 'Remote Protocol :: WebDriver BiDi' + - path: /webdriver + component: 'Remote Protocol :: Marionette' + - path: /beacon + component: 'Core :: Networking' + - path: /permissions + component: 'Core :: DOM: Core & HTML' + - path: /gpc + component: 'Core :: Privacy: Anti-Tracking' + - path: /png + component: 'Core :: Graphics: ImageLib' +- host: html.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' + paths: + - path: /multipage/urls-and-fetching\.html + component: 'Core :: Networking' + - path: /multipage/speculative-loading\.html + component: 'Core :: DOM: Navigation' + - path: /multipage/system-state\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/iframe-embed-object\.html + component: 'Core :: Layout: Images, Video, and HTML Frames' + - path: /multipage/canvas\.html + component: 'Core :: Graphics: Canvas2D' + - path: /multipage/workers\.html + component: 'Core :: DOM: Service Workers' + - path: /multipage/web-messaging\.html + component: 'Core :: DOM: postMessage' + - path: /multipage/server-sent-events\.html + component: 'Core :: Networking' + - path: /multipage/nav-history-apis\.html + component: 'Core :: DOM: Navigation' + - path: /multipage/scripting\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/imagebitmap-and-animations\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/timers-and-user-prompts\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/structured-data\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/dynamic-markup-insertion\.html + component: 'Core :: DOM: Serializers' + - path: /multipage/media\.html + component: 'Core :: Audio/Video: Playback' + - path: /multipage/embedded-content\.html + component: 'Core :: Layout: Images, Video, and HTML Frames' + - path: /multipage/image-maps\.html + component: 'Core :: Layout: Images, Video, and HTML Frames' + - path: /multipage/tables\.html + component: 'Core :: Layout: Tables' + - path: /multipage/dnd\.html + component: 'Core :: DOM: Copy & Paste and Drag & Drop' + - path: /multipage/(forms|form-elements|form-control-infrastructure|input)\.html + component: 'Core :: DOM: Forms' +- host: drafts.css-houdini.org + component: 'Core :: CSS Parsing and Computation' + paths: + - path: /css-paint-api + component: 'Core :: CSS Parsing and Computation' +- host: tc39.es + component: 'Core :: JavaScript Engine' + paths: + - path: /ecma402 + component: 'Core :: JavaScript: Internationalization API' +- host: github.com + paths: + - path: /tc39/ + component: 'Core :: JavaScript Engine' + - path: /WebAssembly/ + component: 'Core :: JavaScript: WebAssembly' + - path: /whatwg/(html|dom) + component: 'Core :: DOM: Core & HTML' + - path: /w3c/manifest + component: 'Firefox :: General' + - path: /WICG/html-in-canvas + component: 'Core :: Graphics: Canvas2D' + - path: /WICG/PEPC + component: 'Core :: DOM: Core & HTML' + - path: /WICG/install-element + component: 'Firefox :: General' + - path: /WICG/privacy-preserving-ads + component: 'Core :: Privacy: Anti-Tracking' + - path: /MicrosoftEdge/MSEdgeExplainers/.*OpaqueRange + component: 'Core :: DOM: Forms' + - path: /MicrosoftEdge/MSEdgeExplainers/.*AriaNotify + component: 'Core :: Disability Access APIs' + - path: /MicrosoftEdge/MSEdgeExplainers/.*DocumentSubtitle + component: 'Core :: DOM: Core & HTML' +- host: webassembly.github.io + component: 'Core :: JavaScript: WebAssembly' +- host: registry.khronos.org + component: 'Core :: Graphics: CanvasWebGL' +- host: gpuweb.github.io + component: 'Core :: Graphics: WebGPU' +- host: webaudio.github.io + component: 'Core :: Web Audio' + paths: + - path: /web-speech-api + component: 'Core :: DOM: Device Interfaces' + - path: /web-midi-api + component: 'Core :: DOM: Device Interfaces' +- host: immersive-web.github.io + component: 'Core :: WebVR' +- host: dom.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' +- host: streams.spec.whatwg.org + component: 'Core :: DOM: Streams' +- host: fetch.spec.whatwg.org + component: 'Core :: DOM: Networking' +- host: cookiestore.spec.whatwg.org + component: 'Core :: Networking: Cookies' +- host: url.spec.whatwg.org + component: 'Core :: Networking' +- host: urlpattern.spec.whatwg.org + component: 'Core :: Networking' +- host: xhr.spec.whatwg.org + component: 'Core :: DOM: Networking' +- host: websockets.spec.whatwg.org + component: 'Core :: Networking: WebSockets' +- host: storage.spec.whatwg.org + component: 'Core :: Storage: Quota Manager' +- host: fs.spec.whatwg.org + component: 'Core :: DOM: File' +- host: notifications.spec.whatwg.org + component: 'Core :: DOM: Notifications' +- host: encoding.spec.whatwg.org + component: 'Core :: Internationalization' +- host: compression.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' +- host: console.spec.whatwg.org + component: 'DevTools :: Console' +- host: fullscreen.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' +- host: webidl.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' +- host: compat.spec.whatwg.org + component: 'Core :: CSS Parsing and Computation' +- host: httpwg.org + component: 'Core :: Networking: HTTP' + paths: + - path: /specs/rfc9842 + component: 'Core :: Networking: Cache' +- host: www.rfc-editor.org + component: 'Core :: Networking' +- host: w3c-fedid.github.io + component: 'Core :: DOM: Credential Management' +- host: w3c-cg.github.io + paths: + - path: /web-nfc + component: 'Core :: DOM: Device Interfaces' +- host: webbluetoothcg.github.io + component: 'Core :: DOM: Device Interfaces' +- host: wicg.github.io + component: 'Core :: DOM: Core & HTML' + paths: + - path: /attribution-reporting + component: 'Core :: Privacy: Anti-Tracking' + - path: /first-party-sets + component: 'Core :: Privacy: Anti-Tracking' + - path: /turtledove + component: 'Core :: Privacy: Anti-Tracking' + - path: /shared-storage + component: 'Core :: Privacy: Anti-Tracking' + - path: /manifest-incubations + component: 'Firefox :: General' + - path: /web-app-launch + component: 'Firefox :: General' + - path: /get-installed-related-apps + component: 'Firefox :: General' + - path: /(local-network-access|private-network-access|netinfo|savedata) + component: 'Core :: DOM: Networking' + - path: /ua-client-hints + component: 'Core :: Networking: HTTP' + - path: /(is-input-pending|performance-measure-memory|js-self-profiling|scheduling-apis) + component: 'Core :: DOM: Performance APIs' + - path: /layout-instability + component: 'Core :: Layout' + - path: /storage-buckets + component: 'Core :: Storage: Quota Manager' + - path: /(background-fetch|background-sync|periodic-background-sync|content-index) + component: 'Core :: DOM: Service Workers' + - path: /sanitizer-api + component: 'Core :: DOM: Security' + - path: /signature-based-sri + component: 'Core :: DOM: Security' + - path: /webcrypto-secure-curves + component: 'Core :: DOM: Security' + - path: /web-otp + component: 'Core :: DOM: Web Authentication' + - path: /(serial|webhid|webusb|shape-detection-api|idle-detection) + component: 'Core :: DOM: Device Interfaces' + - path: /keyboard-(lock|map) + component: 'Core :: DOM: UI Events & Focus Handling' + - path: /ink-enhancement + component: 'Core :: DOM: UI Events & Focus Handling' + - path: /entries-api + component: 'Core :: DOM: File' + - path: /eyedropper-api + component: 'Core :: DOM: Core & HTML' + - path: /local-font-access + component: 'Core :: Layout: Text and Fonts' + - path: /(crash-reporting|deprecation-reporting|intervention-reporting) + component: 'Core :: DOM: Core & HTML' + - path: /nav-speculation + component: 'Core :: DOM: Navigation' + - path: /scroll-to-text-fragment + component: 'Core :: DOM: Navigation' + - path: /video-rvfc + component: 'Core :: Audio/Video: Playback' + - path: /document-picture-in-picture + component: 'Core :: DOM: Core & HTML' + - path: /controls-list + component: 'Core :: Audio/Video: Playback' + - path: /file-system-access + component: 'WebExtensions :: General' + - path: /digital-goods + component: 'Core :: DOM: Web Payments' + - path: /anonymous-iframe + component: 'Core :: DOM: Core & HTML' + - path: /fenced-frame + component: 'Core :: Privacy: Anti-Tracking' + - path: /portals + component: 'Core :: DOM: Core & HTML' + - path: /PEPC + component: 'Core :: DOM: Core & HTML' + - path: /window-controls-overlay + component: 'Firefox :: General' + - path: /observable + component: 'Core :: DOM: Core & HTML' + - path: /page-lifecycle + component: 'Core :: DOM: Core & HTML' + - path: /permissions-request + component: 'Core :: DOM: Core & HTML' +- host: privacycg.github.io + component: 'Core :: Privacy: Anti-Tracking' +- host: patcg-individual-drafts.github.io + component: 'Core :: Privacy: Anti-Tracking' +- host: www.iso.org + paths: + - path: /standard/ + component: 'Core :: Graphics: ImageLib' +- host: jpeg.org + component: 'Core :: Graphics: ImageLib' +- host: aomediacodec.github.io + component: 'Core :: Graphics: ImageLib' + paths: + - path: /av1-rtp + component: 'Core :: WebRTC: Signaling' +- host: svgwg.org + component: 'Core :: SVG' +- host: webmachinelearning.github.io + component: 'Core :: DOM: Core & HTML' +- host: screen-share.github.io + component: 'Core :: WebRTC: Audio/Video' +- host: open-ui.org + component: 'Core :: DOM: Core & HTML' +- host: www.w3.org + component: 'Core :: DOM: Core & HTML' + paths: + - path: /TR/device-memory + component: 'Core :: DOM: Core & HTML' + - path: /TR/.*feature-policy + component: 'Core :: DOM: Core & HTML' + - path: /TR/webnn + component: 'Core :: DOM: Core & HTML' + - path: /TR/webauthn + component: 'Core :: DOM: Web Authentication' + - path: /TR/webtransport + component: 'Core :: Networking' + - path: /TR/webrtc + component: 'Core :: WebRTC' + - path: /TR/mediasession + component: 'Core :: Audio/Video: Playback' + - path: /TR/media-capabilities + component: 'Core :: Audio/Video: Playback' + - path: /TR/media-frags + component: 'Core :: Graphics: ImageLib' + - path: /TR/discovery-api + component: 'Core :: Networking' + - path: /TR/png + component: 'Core :: Graphics: ImageLib' + - path: /Graphics/GIF + component: 'Core :: Graphics: ImageLib' + - path: /Graphics/SVG + component: 'Core :: SVG' + - path: /TR/SVG + component: 'Core :: SVG' + - path: /TR/SMIL + component: 'Core :: SVG' + - path: '[Mm]ath[Mm][Ll]' + component: 'Core :: MathML' + - path: (xslt|xpath) + component: 'Core :: XSLT' + - path: REC-xml-names + component: 'Core :: XSLT' + - path: xml-entity-names + component: 'Core :: DOM: HTML Parser' + - path: (xhtml11|\.dtd|xmlschema|/swap/) + component: 'Core :: XML' + - path: /International + component: 'Core :: Layout: Text and Fonts' + - path: /WAI/ + component: 'Core :: Disability Access APIs' + - path: /TR/DOM-Level-2-Style + component: 'Core :: DOM: CSS Object Model' + - path: /TR/DOM-Level-2/events + component: 'Core :: DOM: Events' + - path: the-canvas-element + component: 'Core :: Graphics: Canvas2D' + - path: /TR/.*selectors- + component: 'Core :: CSS Parsing and Computation' + - path: (box|visuren)\.html + component: 'Core :: Layout: Block and Inline' + - path: tables + component: 'Core :: Layout: Tables' + - path: scroll-snap + component: 'Core :: Layout: Scrolling and Overflow' + - path: css3?-break + component: 'Core :: Layout' + - path: css3-text + component: 'Core :: Layout: Text and Fonts' + - path: /TR/.*html5 + component: 'Core :: DOM: Core & HTML' + - path: '[Cc][Ss][Ss]' + component: 'Core :: CSS Parsing and Computation' diff --git a/templates/web_platform_features.html b/templates/web_platform_features.html index 1eba74e32..c4ddc259d 100644 --- a/templates/web_platform_features.html +++ b/templates/web_platform_features.html @@ -10,7 +10,7 @@ - {% for i, (bugid, summary, changes, whiteboard, user_story) in enumerate(data) -%} + {% for i, (bugid, summary, change_type, changes, whiteboard, user_story) in enumerate(data) -%} @@ -19,84 +19,146 @@ {{ summary | e }} - + {% elif change_type == "create" %} +

+ Created bug: +

+ {% endif -%} + + + {% endfor -%} + diff --git a/tests/rules/test_web_platform_features.py b/tests/rules/test_web_platform_features.py index 2ba6f9311..97c2ab1a4 100644 --- a/tests/rules/test_web_platform_features.py +++ b/tests/rules/test_web_platform_features.py @@ -1,5 +1,6 @@ from bugbot.rules.web_platform_features import ( AddRemoveChange, + BugzillaNewBug, BugzillaUpdate, FeatureBugUpdate, Resolution, @@ -143,7 +144,7 @@ def test_feature_bug_update_into_bugzilla_update(): ) == BugzillaUpdate(resolution="", status="REOPENED") -def test_handlebug(): +def test_handlebug_update(): cleaner = WebPlatformFeatures() cleaner.bug_updates = {1234: FeatureBugUpdate(keywords={"add-keyword": True})} data = {} @@ -165,6 +166,43 @@ def test_handlebug(): "changes": BugzillaUpdate(keywords=AddRemoveChange(add=["add-keyword"])), "whiteboard": "", "user_story": "web-feature: test", + "change_type": "update", } assert output_bug == input_bug assert cleaner.autofix_changes == {"1234": {"keywords": {"add": ["add-keyword"]}}} + + +def test_get_bugs_create(): + cleaner = WebPlatformFeatures() + bug = BugzillaNewBug( + summary="Test bug", + product="Test product", + component="Test component", + description="Test description", + type="enhancement", + keywords=["web-feature"], + whiteboard="[test]", + see_also=["https://example.org"], + user_story="A test user_story", + url="https://example.test", + ) + cleaner.dryrun = True + cleaner.test_mode = True + cleaner.create_bugs = {"feature-id": bug} + bugs = cleaner.get_bugs() + + assert bugs["0"] == { + "id": 0, + "summary": bug.summary, + "url": bug.url, + "see_also": bug.component, + "keywords": bug.keywords, + "whiteboard": bug.whiteboard, + "cf_user_story": bug.user_story, + "status": "NEW", + "resolution": "", + "change_type": "create", + "changes": bug, + "user_story": bug.user_story, + } + assert cleaner.bugs_created[0] == bug diff --git a/tests/test_spec_mapping.py b/tests/test_spec_mapping.py new file mode 100644 index 000000000..715a6e7ee --- /dev/null +++ b/tests/test_spec_mapping.py @@ -0,0 +1,52 @@ +import pytest + +from bugbot import spec_mapping + + +@pytest.fixture(scope="session") +def spec_mapper(): + return spec_mapping.SpecMapper.load() + + +@pytest.mark.parametrize( + ["urls", "product", "component"], + [ + ( + ["https://drafts.csswg.org/css-fonts-4#foo"], + "Core", + "Layout: Text and Fonts", + ), + ( + [ + "https://drafts.csswg.org/some-default-path", + ], + "Core", + "Layout: General", + ), + ( + [ + "https://example.org", + ], + "Core", + "General", + ), + ( + [ + "https://drafts.csswg.org/css-fonts-4#foo", + "https://drafts.csswg.org/css-values-2#foo", + ], + "Core", + "CSS Parsing and Computation", + ), + ( + [ + "https://w3c.github.io/web-share", + "https://drafts.csswg.org/some-default-path", + ], + "Core", + "DOM: Web Share", + ), + ], +) +def test_map_urls(spec_mapper, urls, product, component): + assert spec_mapper.map_urls(urls) == (product, component)