From 0f70140b24fc46667822a882f11c3cd6ef42e151 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 29 May 2026 16:30:51 +0100 Subject: [PATCH 1/2] Add module for best-guess mapping spec URLs to bugzilla product/component This is useful for filing new bugs for incoming feature requests where we might have one or more spec URLs and want to make an initial guess as to where the bug should live. The initial lookup tree was built using Claude to map web-feature bugs with a known spec URL to the relevant component. This means it's likely to be reliable for future features that exist in existing specs but may be wrong for new specs which don't cleanly fit onto existing patterns. Ultimately it might make more sense to store this data somewhere outside of bugbot so that it's more easily kept up to date. --- bugbot/spec_mapping.py | 155 ++++++++++ bugbot/spec_mapping.yml | 573 +++++++++++++++++++++++++++++++++++++ tests/test_spec_mapping.py | 52 ++++ 3 files changed, 780 insertions(+) create mode 100644 bugbot/spec_mapping.py create mode 100644 bugbot/spec_mapping.yml create mode 100644 tests/test_spec_mapping.py diff --git a/bugbot/spec_mapping.py b/bugbot/spec_mapping.py new file mode 100644 index 000000000..65a9c1a6c --- /dev/null +++ b/bugbot/spec_mapping.py @@ -0,0 +1,155 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# mypy: disallow-untyped-defs +"""Map web specification URLs to Bugzilla (product, component) pairs.""" + +import itertools +import os +import re +from abc import ABC, abstractmethod +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from typing import Self +from urllib.parse import SplitResult, urlsplit + +import yaml + +ProductComponent = tuple[str, str] + + +@dataclass +class RuleMatch: + """A resolved (product, component)""" + + position: tuple[int, int] + product: str + component: str + + def __gt__(self, other: "RuleMatch") -> bool: + return (self.position[0], -self.position[1]) > ( + other.position[0], + -other.position[1], + ) + + +_order = itertools.count() + + +class Rule(ABC): + level: int = 0 + + def __init__( + self, + component: ProductComponent | None = None, + paths: Sequence["Rule"] = (), + ): + self.component = component + self.order = next(_order) + + @abstractmethod + def get(self, parsed: SplitResult) -> RuleMatch | None: + ... + + +class HostRule(Rule): + level = 0 + + def __init__( + self, + component: ProductComponent | None, + paths: Sequence["PathRule"] = (), + ): + super().__init__(component) + self.paths = paths + + def get(self, parsed: SplitResult) -> RuleMatch | None: + for path_rule in self.paths: + m = path_rule.get(parsed) + if m is not None: + return m + if self.component is not None: + return RuleMatch((self.level, self.order), *self.component) + return None + + +class PathRule(Rule): + level = 1 + + def __init__( + self, + path: str | None, + component: ProductComponent, + ): + self.path = re.compile(path) if path is not None else None + super().__init__(component) + + def get(self, parsed: SplitResult) -> RuleMatch | None: + if self.path is None or self.path.search(parsed.path) is not None: + assert self.component is not None + return RuleMatch((self.level, self.order), *self.component) + return None + + +def parse_component(value: str) -> ProductComponent: + if " :: " not in value: + raise ValueError(f"Invalid component '{value}'") + product, _, component = value.partition(" :: ") + return product, component + + +class SpecMapper: + def __init__(self, rules: dict[str, HostRule], default: ProductComponent): + self.rules = rules + self.default = default + + @classmethod + def load( + cls, path: str = os.path.join(os.path.dirname(__file__), "spec_mapping.yml") + ) -> Self: + with open(path) as f: + data = yaml.safe_load(f) + rules: dict[str, HostRule] = {} + default: ProductComponent | None = None + for entry in data: + component = entry.get("component") + if "host" not in entry: + if default is not None: + raise ValueError( + f"Got multiple default components second in entry {entry}" + ) + default = parse_component(component) + continue + paths = [ + PathRule(path_entry["path"], parse_component(path_entry["component"])) + for path_entry in entry.get("paths", []) + ] + rules[entry["host"]] = HostRule( + parse_component(component) if component is not None else None, + paths, + ) + if default is None: + raise ValueError("Missing default component") + return cls(rules, default) + + def map_url(self, url: str) -> RuleMatch | None: + """Return the most specific :class:`Match` for ``url``, or ``None``.""" + parsed = urlsplit(url) + host_rule = self.rules.get(parsed.hostname or "") + if host_rule is None: + return None + return host_rule.get(parsed) + + def map_urls(self, urls: Iterable[str]) -> ProductComponent: + """Pick a plausible (product, component) for a feature with given spec URLs.""" + best_match: RuleMatch | None = None + for url in urls: + rule_match = self.map_url(url) + if rule_match is None: + continue + if best_match is None or rule_match > best_match: + best_match = rule_match + if best_match is None: + return self.default + return best_match.product, best_match.component diff --git a/bugbot/spec_mapping.yml b/bugbot/spec_mapping.yml new file mode 100644 index 000000000..816bec910 --- /dev/null +++ b/bugbot/spec_mapping.yml @@ -0,0 +1,573 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# Maps web specification hosts/paths to Bugzilla "Product :: Component". +# Each entry matches a URL host; `paths` sub-rules match the URL path +# (as a regular expression) and take precedence over the host `component`. + +- component: 'Core :: General' +- host: drafts.csswg.org + component: 'Core :: Layout: General' + paths: + - path: /css-values- + component: 'Core :: CSS Parsing and Computation' + - path: /css-cascade- + component: 'Core :: CSS Parsing and Computation' + - path: /css-color-(\d|adjust|hdr|6) + component: 'Core :: CSS Parsing and Computation' + - path: /css-conditional- + component: 'Core :: CSS Parsing and Computation' + - path: /css-view-transitions- + component: 'Core :: CSS Parsing and Computation' + - path: /css-highlight-api- + component: 'Core :: CSS Parsing and Computation' + - path: /mediaqueries- + component: 'Core :: CSS Parsing and Computation' + - path: /css-viewport + component: 'Core :: CSS Parsing and Computation' + - path: /css-anchor-position- + component: 'Core :: CSS Parsing and Computation' + - path: /css-position- + component: 'Core :: CSS Parsing and Computation' + - path: /css-sizing- + component: 'Core :: CSS Parsing and Computation' + - path: /css-display- + component: 'Core :: CSS Parsing and Computation' + - path: /css-contain- + component: 'Core :: CSS Parsing and Computation' + - path: /css-variables- + component: 'Core :: CSS Parsing and Computation' + - path: /css-mixins- + component: 'Core :: CSS Parsing and Computation' + - path: /css-nesting- + component: 'Core :: CSS Parsing and Computation' + - path: /css-namespaces- + component: 'Core :: CSS Parsing and Computation' + - path: /selectors- + component: 'Core :: CSS Parsing and Computation' + - path: /css-pseudo- + component: 'Core :: CSS Parsing and Computation' + - path: /css-inline- + component: 'Core :: Layout: Block and Inline' + - path: /css-break- + component: 'Core :: Layout: Block and Inline' + - path: /css-text- + component: 'Core :: Layout: Text and Fonts' + - path: /css-fonts- + component: 'Core :: Layout: Text and Fonts' + - path: /css-text-decor- + component: 'Core :: Layout: Text and Fonts' + - path: /css-writing-modes- + component: 'Core :: Layout: Text and Fonts' + - path: /css-size-adjust- + component: 'Core :: Layout: Text and Fonts' + - path: /css-rhythm- + component: 'Core :: Layout: Text and Fonts' + - path: /css-counter-styles + component: 'Core :: Layout: Text and Fonts' + - path: /css-tables- + component: 'Core :: Layout: Tables' + - path: /css-grid- + component: 'Core :: Layout: Grid' + - path: /css-flexbox- + component: 'Core :: Layout' + - path: /css-multicol + component: 'Core :: Layout' + - path: /css-ruby- + component: 'Core :: Layout: Ruby' + - path: /css-scroll- + component: 'Core :: Layout: Scrolling and Overflow' + - path: /css-overflow- + component: 'Core :: Layout: Scrolling and Overflow' + - path: /css-overscroll + component: 'Core :: Layout: Scrolling and Overflow' + - path: /scroll-animations + component: 'Core :: DOM: Animation' + - path: /css-images- + component: 'Core :: Layout: Images, Video, and HTML Frames' + - path: /css-shapes- + component: 'Core :: Layout' + - path: /css-masking- + component: 'Core :: SVG' + - path: /filter-effects- + component: 'Core :: SVG' + - path: /compositing- + component: 'Core :: Web Painting' + - path: /geometry- + component: 'Core :: DOM: CSS Object Model' + - path: /css-page- + component: 'Core :: Printing: Output' + - path: /css-animations- + component: 'Core :: CSS Transitions and Animations' + - path: /css-transitions- + component: 'Core :: CSS Transitions and Animations' + - path: /css-easing- + component: 'Core :: CSS Transitions and Animations' + - path: /css-transforms- + component: 'Core :: CSS Transitions and Animations' + - path: /web-animations- + component: 'Core :: DOM: Animation' + - path: /motion- + component: 'Core :: DOM: Animation' + - path: /css-will-change + component: 'Core :: Web Painting' + - path: /css-shadow + component: 'Core :: Web Painting' + - path: /cssom-view + component: 'Core :: Layout' + - path: /cssom + component: 'Core :: DOM: CSS Object Model' + - path: /resize-observer + component: 'Core :: Layout' + - path: /css-gaps- + component: 'Core :: Layout' + - path: /css-box- + component: 'Core :: Layout' + - path: /css-align- + component: 'Core :: Layout' + - path: /css-logical- + component: 'Core :: Layout' + - path: /css-env- + component: 'Core :: Layout' + - path: /css-forms- + component: 'Core :: Layout' + - path: /css-ui- + component: 'Core :: Layout' + - path: /css-speech- + component: 'Core :: Disability Access APIs' + - path: /mathml + component: 'Core :: MathML' +- host: w3c.github.io + component: 'Core :: DOM: Core & HTML' + paths: + - path: /manifest + component: 'Firefox :: General' + - path: /badging + component: 'Firefox for Android :: PWA' + - path: /web-share-target + component: 'Core :: DOM: Web Share' + - path: /web-share + component: 'Core :: DOM: Web Share' + - path: /(accelerometer|ambient-light|gyroscope|magnetometer|orientation-sensor|sensors|compute-pressure|deviceorientation|device-posture|battery) + component: 'Core :: DOM: Device Interfaces' + - path: /aria + component: 'Core :: Disability Access APIs' + - path: /clipboard-apis + component: 'Core :: DOM: Copy & Paste and Drag & Drop' + - path: /editing + component: 'Core :: DOM: Editor' + - path: /edit-context + component: 'Core :: DOM: Editor' + - path: /webappsec-permissions-policy + component: 'Core :: DOM: Core & HTML' + - path: /webappsec-credential-management + component: 'Core :: DOM: Credential Management' + - path: /webappsec-referrer-policy + component: 'Core :: Networking' + - path: /webappsec- + component: 'Core :: DOM: Security' + - path: /trusted-types + component: 'Core :: DOM: Security' + - path: /webcrypto + component: 'Core :: DOM: Security' + - path: /(hr-time|performance-timeline|user-timing|server-timing|navigation-timing|resource-timing|element-timing|event-timing|paint-timing|largest-contentful-paint|long-animation-frames) + component: 'Core :: DOM: Performance APIs' + - path: /(requestidlecallback|longtasks) + component: 'Core :: DOM: Core & HTML' + - path: /IntersectionObserver + component: 'Core :: Layout' + - path: /IndexedDB + component: 'Core :: Storage: IndexedDB' + - path: /FileAPI + component: 'Core :: DOM: File' + - path: /web-locks + component: 'Core :: DOM: Core & HTML' + - path: /mediacapture-transform + component: 'Core :: WebRTC: Audio/Video' + - path: /mediacapture- + component: 'Core :: WebRTC: Audio/Video' + - path: /webrtc + component: 'Core :: WebRTC' + - path: /webtransport + component: 'Core :: Networking' + - path: /audio-session + component: 'Core :: Audio/Video: Playback' + - path: /encrypted-media + component: 'Core :: Audio/Video: Playback' + - path: /media-(capabilities|playback-quality|source) + component: 'Core :: Audio/Video: Playback' + - path: /mediasession + component: 'Core :: Audio/Video: Playback' + - path: /webcodecs + component: 'Core :: Audio/Video: Web Codecs' + - path: /remote-playback + component: 'Core :: Audio/Video' + - path: /webvtt + component: 'Core :: Audio/Video: Playback' + - path: /html-media-capture + component: 'Core :: Audio/Video: Recording' + - path: /picture-in-picture + component: 'Core :: DOM: Core & HTML' + - path: /mathml + component: 'Core :: MathML' + - path: /svgwg + component: 'Core :: SVG' + - path: /ServiceWorker + component: 'Core :: DOM: Service Workers' + - path: /push-api + component: 'Core :: DOM: Push Subscriptions' + - path: /gamepad + component: 'Core :: DOM: Device Interfaces' + - path: /geolocation + component: 'Core :: DOM: Geolocation' + - path: /(pointerevents|pointerlock|touch-events|uievents) + component: 'Core :: DOM: UI Events & Focus Handling' + - path: /vibration + component: 'Core :: DOM: Device Interfaces' + - path: /screen-orientation + component: 'Core :: DOM: Device Interfaces' + - path: /screen-wake-lock + component: 'Core :: DOM: Device Interfaces' + - path: /virtual-keyboard + component: 'Core :: Layout' + - path: /window-management + component: 'Core :: DOM: Core & HTML' + - path: /(payment-request|web-based-payment-handler|secure-payment-confirmation) + component: 'Core :: DOM: Web Payments' + - path: /webauthn + component: 'Core :: DOM: Web Authentication' + - path: /selection-api + component: 'Core :: DOM: Core & HTML' + - path: /DOM-Parsing + component: 'Core :: DOM: Serializers' + - path: /reporting + component: 'Core :: DOM: Core & HTML' + - path: /presentation-api + component: 'Core :: DOM: Core & HTML' + - path: /webdriver-bidi + component: 'Remote Protocol :: WebDriver BiDi' + - path: /webdriver + component: 'Remote Protocol :: Marionette' + - path: /beacon + component: 'Core :: Networking' + - path: /permissions + component: 'Core :: DOM: Core & HTML' + - path: /gpc + component: 'Core :: Privacy: Anti-Tracking' + - path: /png + component: 'Core :: Graphics: ImageLib' +- host: html.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' + paths: + - path: /multipage/urls-and-fetching\.html + component: 'Core :: Networking' + - path: /multipage/speculative-loading\.html + component: 'Core :: DOM: Navigation' + - path: /multipage/system-state\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/iframe-embed-object\.html + component: 'Core :: Layout: Images, Video, and HTML Frames' + - path: /multipage/canvas\.html + component: 'Core :: Graphics: Canvas2D' + - path: /multipage/workers\.html + component: 'Core :: DOM: Service Workers' + - path: /multipage/web-messaging\.html + component: 'Core :: DOM: postMessage' + - path: /multipage/server-sent-events\.html + component: 'Core :: Networking' + - path: /multipage/nav-history-apis\.html + component: 'Core :: DOM: Navigation' + - path: /multipage/scripting\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/imagebitmap-and-animations\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/timers-and-user-prompts\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/structured-data\.html + component: 'Core :: DOM: Core & HTML' + - path: /multipage/dynamic-markup-insertion\.html + component: 'Core :: DOM: Serializers' + - path: /multipage/media\.html + component: 'Core :: Audio/Video: Playback' + - path: /multipage/embedded-content\.html + component: 'Core :: Layout: Images, Video, and HTML Frames' + - path: /multipage/image-maps\.html + component: 'Core :: Layout: Images, Video, and HTML Frames' + - path: /multipage/tables\.html + component: 'Core :: Layout: Tables' + - path: /multipage/dnd\.html + component: 'Core :: DOM: Copy & Paste and Drag & Drop' + - path: /multipage/(forms|form-elements|form-control-infrastructure|input)\.html + component: 'Core :: DOM: Forms' +- host: drafts.css-houdini.org + component: 'Core :: CSS Parsing and Computation' + paths: + - path: /css-paint-api + component: 'Core :: CSS Parsing and Computation' +- host: tc39.es + component: 'Core :: JavaScript Engine' + paths: + - path: /ecma402 + component: 'Core :: JavaScript: Internationalization API' +- host: github.com + paths: + - path: /tc39/ + component: 'Core :: JavaScript Engine' + - path: /WebAssembly/ + component: 'Core :: JavaScript: WebAssembly' + - path: /whatwg/(html|dom) + component: 'Core :: DOM: Core & HTML' + - path: /w3c/manifest + component: 'Firefox :: General' + - path: /WICG/html-in-canvas + component: 'Core :: Graphics: Canvas2D' + - path: /WICG/PEPC + component: 'Core :: DOM: Core & HTML' + - path: /WICG/install-element + component: 'Firefox :: General' + - path: /WICG/privacy-preserving-ads + component: 'Core :: Privacy: Anti-Tracking' + - path: /MicrosoftEdge/MSEdgeExplainers/.*OpaqueRange + component: 'Core :: DOM: Forms' + - path: /MicrosoftEdge/MSEdgeExplainers/.*AriaNotify + component: 'Core :: Disability Access APIs' + - path: /MicrosoftEdge/MSEdgeExplainers/.*DocumentSubtitle + component: 'Core :: DOM: Core & HTML' +- host: webassembly.github.io + component: 'Core :: JavaScript: WebAssembly' +- host: registry.khronos.org + component: 'Core :: Graphics: CanvasWebGL' +- host: gpuweb.github.io + component: 'Core :: Graphics: WebGPU' +- host: webaudio.github.io + component: 'Core :: Web Audio' + paths: + - path: /web-speech-api + component: 'Core :: DOM: Device Interfaces' + - path: /web-midi-api + component: 'Core :: DOM: Device Interfaces' +- host: immersive-web.github.io + component: 'Core :: WebVR' +- host: dom.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' +- host: streams.spec.whatwg.org + component: 'Core :: DOM: Streams' +- host: fetch.spec.whatwg.org + component: 'Core :: DOM: Networking' +- host: cookiestore.spec.whatwg.org + component: 'Core :: Networking: Cookies' +- host: url.spec.whatwg.org + component: 'Core :: Networking' +- host: urlpattern.spec.whatwg.org + component: 'Core :: Networking' +- host: xhr.spec.whatwg.org + component: 'Core :: DOM: Networking' +- host: websockets.spec.whatwg.org + component: 'Core :: Networking: WebSockets' +- host: storage.spec.whatwg.org + component: 'Core :: Storage: Quota Manager' +- host: fs.spec.whatwg.org + component: 'Core :: DOM: File' +- host: notifications.spec.whatwg.org + component: 'Core :: DOM: Notifications' +- host: encoding.spec.whatwg.org + component: 'Core :: Internationalization' +- host: compression.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' +- host: console.spec.whatwg.org + component: 'DevTools :: Console' +- host: fullscreen.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' +- host: webidl.spec.whatwg.org + component: 'Core :: DOM: Core & HTML' +- host: compat.spec.whatwg.org + component: 'Core :: CSS Parsing and Computation' +- host: httpwg.org + component: 'Core :: Networking: HTTP' + paths: + - path: /specs/rfc9842 + component: 'Core :: Networking: Cache' +- host: www.rfc-editor.org + component: 'Core :: Networking' +- host: w3c-fedid.github.io + component: 'Core :: DOM: Credential Management' +- host: w3c-cg.github.io + paths: + - path: /web-nfc + component: 'Core :: DOM: Device Interfaces' +- host: webbluetoothcg.github.io + component: 'Core :: DOM: Device Interfaces' +- host: wicg.github.io + component: 'Core :: DOM: Core & HTML' + paths: + - path: /attribution-reporting + component: 'Core :: Privacy: Anti-Tracking' + - path: /first-party-sets + component: 'Core :: Privacy: Anti-Tracking' + - path: /turtledove + component: 'Core :: Privacy: Anti-Tracking' + - path: /shared-storage + component: 'Core :: Privacy: Anti-Tracking' + - path: /manifest-incubations + component: 'Firefox :: General' + - path: /web-app-launch + component: 'Firefox :: General' + - path: /get-installed-related-apps + component: 'Firefox :: General' + - path: /(local-network-access|private-network-access|netinfo|savedata) + component: 'Core :: DOM: Networking' + - path: /ua-client-hints + component: 'Core :: Networking: HTTP' + - path: /(is-input-pending|performance-measure-memory|js-self-profiling|scheduling-apis) + component: 'Core :: DOM: Performance APIs' + - path: /layout-instability + component: 'Core :: Layout' + - path: /storage-buckets + component: 'Core :: Storage: Quota Manager' + - path: /(background-fetch|background-sync|periodic-background-sync|content-index) + component: 'Core :: DOM: Service Workers' + - path: /sanitizer-api + component: 'Core :: DOM: Security' + - path: /signature-based-sri + component: 'Core :: DOM: Security' + - path: /webcrypto-secure-curves + component: 'Core :: DOM: Security' + - path: /web-otp + component: 'Core :: DOM: Web Authentication' + - path: /(serial|webhid|webusb|shape-detection-api|idle-detection) + component: 'Core :: DOM: Device Interfaces' + - path: /keyboard-(lock|map) + component: 'Core :: DOM: UI Events & Focus Handling' + - path: /ink-enhancement + component: 'Core :: DOM: UI Events & Focus Handling' + - path: /entries-api + component: 'Core :: DOM: File' + - path: /eyedropper-api + component: 'Core :: DOM: Core & HTML' + - path: /local-font-access + component: 'Core :: Layout: Text and Fonts' + - path: /(crash-reporting|deprecation-reporting|intervention-reporting) + component: 'Core :: DOM: Core & HTML' + - path: /nav-speculation + component: 'Core :: DOM: Navigation' + - path: /scroll-to-text-fragment + component: 'Core :: DOM: Navigation' + - path: /video-rvfc + component: 'Core :: Audio/Video: Playback' + - path: /document-picture-in-picture + component: 'Core :: DOM: Core & HTML' + - path: /controls-list + component: 'Core :: Audio/Video: Playback' + - path: /file-system-access + component: 'WebExtensions :: General' + - path: /digital-goods + component: 'Core :: DOM: Web Payments' + - path: /anonymous-iframe + component: 'Core :: DOM: Core & HTML' + - path: /fenced-frame + component: 'Core :: Privacy: Anti-Tracking' + - path: /portals + component: 'Core :: DOM: Core & HTML' + - path: /PEPC + component: 'Core :: DOM: Core & HTML' + - path: /window-controls-overlay + component: 'Firefox :: General' + - path: /observable + component: 'Core :: DOM: Core & HTML' + - path: /page-lifecycle + component: 'Core :: DOM: Core & HTML' + - path: /permissions-request + component: 'Core :: DOM: Core & HTML' +- host: privacycg.github.io + component: 'Core :: Privacy: Anti-Tracking' +- host: patcg-individual-drafts.github.io + component: 'Core :: Privacy: Anti-Tracking' +- host: www.iso.org + paths: + - path: /standard/ + component: 'Core :: Graphics: ImageLib' +- host: jpeg.org + component: 'Core :: Graphics: ImageLib' +- host: aomediacodec.github.io + component: 'Core :: Graphics: ImageLib' + paths: + - path: /av1-rtp + component: 'Core :: WebRTC: Signaling' +- host: svgwg.org + component: 'Core :: SVG' +- host: webmachinelearning.github.io + component: 'Core :: DOM: Core & HTML' +- host: screen-share.github.io + component: 'Core :: WebRTC: Audio/Video' +- host: open-ui.org + component: 'Core :: DOM: Core & HTML' +- host: www.w3.org + component: 'Core :: DOM: Core & HTML' + paths: + - path: /TR/device-memory + component: 'Core :: DOM: Core & HTML' + - path: /TR/.*feature-policy + component: 'Core :: DOM: Core & HTML' + - path: /TR/webnn + component: 'Core :: DOM: Core & HTML' + - path: /TR/webauthn + component: 'Core :: DOM: Web Authentication' + - path: /TR/webtransport + component: 'Core :: Networking' + - path: /TR/webrtc + component: 'Core :: WebRTC' + - path: /TR/mediasession + component: 'Core :: Audio/Video: Playback' + - path: /TR/media-capabilities + component: 'Core :: Audio/Video: Playback' + - path: /TR/media-frags + component: 'Core :: Graphics: ImageLib' + - path: /TR/discovery-api + component: 'Core :: Networking' + - path: /TR/png + component: 'Core :: Graphics: ImageLib' + - path: /Graphics/GIF + component: 'Core :: Graphics: ImageLib' + - path: /Graphics/SVG + component: 'Core :: SVG' + - path: /TR/SVG + component: 'Core :: SVG' + - path: /TR/SMIL + component: 'Core :: SVG' + - path: '[Mm]ath[Mm][Ll]' + component: 'Core :: MathML' + - path: (xslt|xpath) + component: 'Core :: XSLT' + - path: REC-xml-names + component: 'Core :: XSLT' + - path: xml-entity-names + component: 'Core :: DOM: HTML Parser' + - path: (xhtml11|\.dtd|xmlschema|/swap/) + component: 'Core :: XML' + - path: /International + component: 'Core :: Layout: Text and Fonts' + - path: /WAI/ + component: 'Core :: Disability Access APIs' + - path: /TR/DOM-Level-2-Style + component: 'Core :: DOM: CSS Object Model' + - path: /TR/DOM-Level-2/events + component: 'Core :: DOM: Events' + - path: the-canvas-element + component: 'Core :: Graphics: Canvas2D' + - path: /TR/.*selectors- + component: 'Core :: CSS Parsing and Computation' + - path: (box|visuren)\.html + component: 'Core :: Layout: Block and Inline' + - path: tables + component: 'Core :: Layout: Tables' + - path: scroll-snap + component: 'Core :: Layout: Scrolling and Overflow' + - path: css3?-break + component: 'Core :: Layout' + - path: css3-text + component: 'Core :: Layout: Text and Fonts' + - path: /TR/.*html5 + component: 'Core :: DOM: Core & HTML' + - path: '[Cc][Ss][Ss]' + component: 'Core :: CSS Parsing and Computation' diff --git a/tests/test_spec_mapping.py b/tests/test_spec_mapping.py new file mode 100644 index 000000000..715a6e7ee --- /dev/null +++ b/tests/test_spec_mapping.py @@ -0,0 +1,52 @@ +import pytest + +from bugbot import spec_mapping + + +@pytest.fixture(scope="session") +def spec_mapper(): + return spec_mapping.SpecMapper.load() + + +@pytest.mark.parametrize( + ["urls", "product", "component"], + [ + ( + ["https://drafts.csswg.org/css-fonts-4#foo"], + "Core", + "Layout: Text and Fonts", + ), + ( + [ + "https://drafts.csswg.org/some-default-path", + ], + "Core", + "Layout: General", + ), + ( + [ + "https://example.org", + ], + "Core", + "General", + ), + ( + [ + "https://drafts.csswg.org/css-fonts-4#foo", + "https://drafts.csswg.org/css-values-2#foo", + ], + "Core", + "CSS Parsing and Computation", + ), + ( + [ + "https://w3c.github.io/web-share", + "https://drafts.csswg.org/some-default-path", + ], + "Core", + "DOM: Web Share", + ), + ], +) +def test_map_urls(spec_mapper, urls, product, component): + assert spec_mapper.map_urls(urls) == (product, component) From 007aff901fc098eeb3c4bb381a23fe115d638d5c Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 29 May 2026 16:28:21 +0100 Subject: [PATCH 2/2] Support creating new bugs for web-features Initially create new bugs only for web-features which are: * Supported in both Chrome and Safari * Not supported in Firefox In practice there shouldn't be any of these, although there are a couple of false positives (features that are incorrectly marked as unsupported in Firefox) that make it a useful test case. The plan is to extend this to file bugs more cases in the future. --- bugbot/rules/web_platform_features.py | 255 ++++++++++++++++++++-- templates/web_platform_features.html | 208 +++++++++++------- tests/rules/test_web_platform_features.py | 40 +++- 3 files changed, 410 insertions(+), 93 deletions(-) diff --git a/bugbot/rules/web_platform_features.py b/bugbot/rules/web_platform_features.py index 36de880c2..41e3bed7f 100644 --- a/bugbot/rules/web_platform_features.py +++ b/bugbot/rules/web_platform_features.py @@ -13,17 +13,21 @@ Any, Generic, Iterable, + Literal, Mapping, MutableMapping, Optional, Sequence, TypeVar, + cast, ) from urllib import parse from google.cloud import bigquery +from libmozdata.bugzilla import Bugzilla +from requests.exceptions import HTTPError -from bugbot import gcp +from bugbot import gcp, logger, spec_mapping, utils from bugbot.bzcleaner import Bug, BzCleaner Json = None | str | int | float | Sequence["Json"] | Mapping[str, "Json"] @@ -87,6 +91,43 @@ def __bool__(self) -> bool: return bool(self.add or self.remove) +@dataclass +class BugzillaNewBug: + """Representation of a new bug to be created""" + + summary: str + product: str + component: str + description: str + type: Literal["defect"] | Literal["enhancement"] | Literal["task"] + version: str = "unspecified" + keywords: Optional[list[str]] = None + whiteboard: Optional[str] = None + see_also: Optional[list[str]] = None + user_story: Optional[str] = None + url: Optional[str] = None + + def to_json(self) -> Mapping[str, Json]: + rv: dict[str, Json] = { + "summary": self.summary, + "product": self.product, + "component": self.component, + "description": self.description, + "version": self.version, + "type": self.type, + } + for value, name in [ + (self.whiteboard, "status_whiteboard"), + (self.keywords, "keywords"), + (self.see_also, "see_also"), + (self.user_story, "cf_user_story"), + (self.url, "url"), + ]: + if value is not None: + rv[name] = value + return rv + + @dataclass class BugzillaUpdate: """Representation of bug changes for use with the Bugzilla ReST API""" @@ -287,11 +328,36 @@ class FeatureData: supported_browsers: set[str] sp_issue: Optional[int] spec_url: set[str] + name: Optional[str] = None + description: Optional[str] = None def is_supported(self) -> bool: return {"firefox", "firefox_android"}.issubset(self.supported_browsers) +def feature_keywords(feature: FeatureData) -> set[str]: + rv = set() + if {"chrome", "chrome_android"}.issubset(feature.supported_browsers): + rv.add("parity-chrome") + if {"safari", "safari_ios"}.issubset(feature.supported_browsers): + rv.add("parity-safari") + return rv + + +def feature_links(feature: FeatureData) -> set[str]: + links = set( + [ + f"https://web-platform-dx.github.io/web-features-explorer/features/{feature.feature}/" + ] + ) + if feature.sp_issue is not None: + links.add( + f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}" + ) + links |= feature.spec_url + return links + + @dataclass class FeatureBug: """Bug that represents a web-feature""" @@ -313,10 +379,7 @@ def expected_keywords(self) -> set[str]: rv.add("web-feature") if not self.is_supported(): for feature in self.features.values(): - if {"chrome", "chrome_android"}.issubset(feature.supported_browsers): - rv.add("parity-chrome") - if {"safari", "safari_ios"}.issubset(feature.supported_browsers): - rv.add("parity-safari") + rv |= feature_keywords(feature) return rv def missing_keywords(self) -> set[str]: @@ -324,15 +387,8 @@ def missing_keywords(self) -> set[str]: def expected_links(self) -> set[str]: links = set() - for feature_name, feature in self.features.items(): - links.add( - f"https://web-platform-dx.github.io/web-features-explorer/features/{feature_name}/" - ) - if feature.sp_issue is not None: - links.add( - f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}" - ) - links |= feature.spec_url + for feature in self.features.values(): + links |= feature_links(feature) return links def missing_links(self) -> set[str]: @@ -366,6 +422,79 @@ def remove_links(self) -> set[str]: _DataType = TypeVar("_DataType") +class CreateRule(ABC, Generic[_DataType]): + """Rule for creating new bugs based on BigQuery data""" + + def __init__(self, client: bigquery.Client): + self.client = client + + @abstractmethod + def get_data(self) -> _DataType: + ... + + @abstractmethod + def create(self, data: _DataType) -> Mapping[str, BugzillaNewBug]: + ... + + def run(self) -> Mapping[str, BugzillaNewBug]: + data: _DataType = self.get_data() + return self.create(data) + + +class FirefoxOnlyMissing(CreateRule): + def get_data(self) -> list[FeatureData]: + query = """ +SELECT + features.feature, + features.name, + features.description, + (SELECT ARRAY_AGG(browser) FROM UNNEST(features.support)) AS supported_browsers, + features.spec as spec_url, + sp_mozilla.issue as sp_issue +FROM `web_features.features_latest` AS features +LEFT JOIN `webcompat_knowledge_base.bugzilla_bugs` AS bugs ON + features.feature IN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature")) +LEFT JOIN `standards_positions.mozilla_standards_positions` AS sp_mozilla ON + sp_mozilla.web_feature = features.feature +WHERE + "safari" in UNNEST(features.support.browser) AND + "chrome" IN UNNEST(features.support.browser) AND + "firefox" NOT IN UNNEST(features.support.browser) AND + bugs.number IS NULL +""" + return [ + FeatureData( + feature=row.feature, + spec_url=set(row.spec_url), + supported_browsers=set(row.supported_browsers), + sp_issue=row.sp_issue, + name=row.name, + description=row.description, + ) + for row in self.client.query(query) + ] + + def create(self, data: list[FeatureData]) -> Mapping[str, BugzillaNewBug]: + rv = {} + spec_mapper = spec_mapping.SpecMapper.load() + for feature in data: + product, component = spec_mapper.map_urls(feature.spec_url) + spec_url = feature.spec_url.pop() + rv[feature.feature] = BugzillaNewBug( + summary=f"[meta] Implement {feature.name}", + product=product, + component=component, + description=f"Implement {feature.name}:\n{feature.description}", + type="enhancement", + keywords=["web-feature"] + list(feature_keywords(feature)), + user_story=f"web-feature: {feature.feature}", + url=spec_url, + see_also=[item for item in feature_links(feature) if item != spec_url], + ) + + return rv + + class UpdateRule(ABC, Generic[_DataType]): """Rule for updating bugs based on BigQuery data""" @@ -598,6 +727,8 @@ def update( class WebPlatformFeatures(BzCleaner): def __init__(self) -> None: super().__init__() + self.create_bugs: dict[str, BugzillaNewBug] = {} + self.bugs_created: dict[int, BugzillaNewBug] = {} self.bug_updates: dict[int, FeatureBugUpdate] = defaultdict(FeatureBugUpdate) def description(self) -> str: @@ -610,24 +741,106 @@ def has_default_products(self) -> bool: return False def columns(self) -> list[str]: - return ["id", "summary", "changes", "whiteboard", "user_story"] + return ["id", "summary", "change_type", "changes", "whiteboard", "user_story"] + + def get_bugs( + self, + date: str = "today", + bug_ids: list[int] = [], + chunk_size: Optional[int] = None, + ) -> dict[str, Mapping[str, Any]]: + bugs = super().get_bugs(date, bug_ids, chunk_size) + + if self.create_bugs: + bugs_for_features = self.get_feature_bugs(set(self.create_bugs.keys())) + else: + bugs_for_features = {} + + for i, (feature, bug) in enumerate(self.create_bugs.items()): + if feature in bugs_for_features: + # A bug was already created for this feature + continue + + if self.dryrun or self.test_mode: + response = {"id": i} + logger.info( + f"A bug '{bug.summary}` would be created with:\n{bug.to_json()}", + ) + else: + try: + response = utils.create_bug(cast(dict, bug.to_json())) + except HTTPError: + logger.error( + f"Failed to create bug '{bug.summary}'", + ) + continue + + bug_id = response["id"] + assert isinstance(bug_id, int) + self.bugs_created[bug_id] = bug + bugs[str(bug_id)] = { + "id": bug_id, + "summary": bug.summary, + "url": bug.url, + "see_also": bug.component, + "keywords": bug.keywords, + "whiteboard": bug.whiteboard, + "cf_user_story": bug.user_story, + "status": "NEW", + "resolution": "", + "change_type": "create", + "changes": bug, + "user_story": bug.user_story, + } + + return bugs + + def get_feature_bugs(self, features: set[str]) -> Mapping[str, int]: + """Get the list of existing bugs for specific features""" + data: dict[str, int] = {} + + def handler(bug: Mapping[str, Any], data: dict[str, int]) -> None: + for _, key, value in parse_user_story(bug["cf_user_story"]): + if key == "web-feature": + if value in features: + data[value] = bug["id"] + + Bugzilla( + { + "keywords": "web-feature", + "keywords_type": "allwords", + "f1": "cf_user_story", + "o1": "substring", + "v1": "web-feature", + "f2": "cf_user_story", + "o2": "anywordssubstr", + "v2": ",".join(features), + }, + bugdata=data, + bughandler=handler, + ).wait() + + return data def handle_bug(self, bug: Bug, data: dict[str, Any]) -> Optional[Bug]: bug_id_str = str(bug["id"]) bug_id_int = int(bug["id"]) - if bug_id_int not in self.bug_updates: - return None - bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug) + if bug_id_int in self.bug_updates: + bugzilla_update = self.bug_updates[bug_id_int].into_bugzilla_update(bug) + if not bugzilla_update: + return None - if bugzilla_update: self.autofix_changes[bug_id_str] = bugzilla_update.to_json() data[bug_id_str] = { + "change_type": "update", "changes": bugzilla_update, "whiteboard": bug["whiteboard"], "user_story": bug["cf_user_story"], } return bug + elif bug_id_int in self.bugs_created: + return bug return None @@ -648,6 +861,10 @@ def get_bz_params(self, date: str) -> dict[str, str | int | list[str] | list[int def get_bug_updates(self) -> None: project = "moz-fx-dev-dschubert-wckb" client = gcp.get_bigquery_client(project, ["cloud-platform", "drive"]) + + for create_rule in [FirefoxOnlyMissing(client)]: + self.create_bugs.update(create_rule.run()) + for update_rule in [ FeatureRenames(client), InvalidFeatures(client), diff --git a/templates/web_platform_features.html b/templates/web_platform_features.html index 1eba74e32..c4ddc259d 100644 --- a/templates/web_platform_features.html +++ b/templates/web_platform_features.html @@ -10,7 +10,7 @@ - {% for i, (bugid, summary, changes, whiteboard, user_story) in enumerate(data) -%} + {% for i, (bugid, summary, change_type, changes, whiteboard, user_story) in enumerate(data) -%} @@ -19,84 +19,146 @@ {{ summary | e }} - + {% elif change_type == "create" %} +

+ Created bug: +

+ {% endif -%} + + + {% endfor -%} + diff --git a/tests/rules/test_web_platform_features.py b/tests/rules/test_web_platform_features.py index 2ba6f9311..97c2ab1a4 100644 --- a/tests/rules/test_web_platform_features.py +++ b/tests/rules/test_web_platform_features.py @@ -1,5 +1,6 @@ from bugbot.rules.web_platform_features import ( AddRemoveChange, + BugzillaNewBug, BugzillaUpdate, FeatureBugUpdate, Resolution, @@ -143,7 +144,7 @@ def test_feature_bug_update_into_bugzilla_update(): ) == BugzillaUpdate(resolution="", status="REOPENED") -def test_handlebug(): +def test_handlebug_update(): cleaner = WebPlatformFeatures() cleaner.bug_updates = {1234: FeatureBugUpdate(keywords={"add-keyword": True})} data = {} @@ -165,6 +166,43 @@ def test_handlebug(): "changes": BugzillaUpdate(keywords=AddRemoveChange(add=["add-keyword"])), "whiteboard": "", "user_story": "web-feature: test", + "change_type": "update", } assert output_bug == input_bug assert cleaner.autofix_changes == {"1234": {"keywords": {"add": ["add-keyword"]}}} + + +def test_get_bugs_create(): + cleaner = WebPlatformFeatures() + bug = BugzillaNewBug( + summary="Test bug", + product="Test product", + component="Test component", + description="Test description", + type="enhancement", + keywords=["web-feature"], + whiteboard="[test]", + see_also=["https://example.org"], + user_story="A test user_story", + url="https://example.test", + ) + cleaner.dryrun = True + cleaner.test_mode = True + cleaner.create_bugs = {"feature-id": bug} + bugs = cleaner.get_bugs() + + assert bugs["0"] == { + "id": 0, + "summary": bug.summary, + "url": bug.url, + "see_also": bug.component, + "keywords": bug.keywords, + "whiteboard": bug.whiteboard, + "cf_user_story": bug.user_story, + "status": "NEW", + "resolution": "", + "change_type": "create", + "changes": bug, + "user_story": bug.user_story, + } + assert cleaner.bugs_created[0] == bug