From 0d7004f4e806cb0ebb5e49899a167b76f6b46c54 Mon Sep 17 00:00:00 2001 From: xrendan Date: Thu, 10 Sep 2026 11:34:57 -0600 Subject: [PATCH 1/2] Fix agent source registration and retire legacy Gemini assessment pipeline The commitment evaluation agent could not register any new evidence: canada.ca rejected the pages endpoint's User-Agent (20s hang then 500), parl.ca was not on the allowlist despite the prompts telling the agent to use it, and the defuddle CLI truncated piped output at 64KB so large pages failed to parse. - Fetch government pages through PageFetcher with a browser-compatible crawler User-Agent; also used by the RSS entry scraper, which hit the same block - Allow *.parl.ca in the pages endpoint; return 502 with a readable message on timeouts, connection errors and parse failures instead of 500 - Replace the local defuddle CLI with the hosted worker (deffudler.svc.canadasbuilding.com), configured via DEFUDDLE_API_KEY and optional DEFUDDLE_API_URL; Rails still fetches the page and posts the HTML - Default both agent jobs to claude-sonnet-5 (were sonnet-4-6 and opus-4-6) - Agent-created commitment matches start as assessed - Remove the 6-hourly CommitmentAssessmentCronJob and the Gemini CriterionAssessor / CommitmentStatusDeriver pipeline, which wrote criteria alongside the agent; drop the rake tasks that drove it - Chat tolerates orphaned STI types so historical rows from the removed classes still load in the admin --- .env.example | 4 + .../agent/commitment_matches_controller.rb | 4 + app/controllers/api/agent/pages_controller.rb | 23 ++-- app/jobs/agent_evaluate_commitment_job.rb | 4 +- app/jobs/agent_process_entry_job.rb | 4 +- app/jobs/commitment_assessment_cron_job.rb | 12 -- app/jobs/commitment_assessment_job.rb | 103 --------------- app/models/chat.rb | 10 ++ app/models/commitment_status_deriver.rb | 120 ------------------ app/models/criterion_assessor.rb | 85 ------------- app/models/entry.rb | 4 +- app/services/page_fetcher.rb | 16 +++ config/initializers/good_job.rb | 6 - docker-compose.prod.yml | 4 + lib/defuddle.rb | 53 +++++--- lib/tasks/commitment_backfill.rake | 15 --- lib/tasks/matching_gaps.rake | 1 - lib/tasks/reevaluate.rake | 117 ----------------- 18 files changed, 96 insertions(+), 489 deletions(-) delete mode 100644 app/jobs/commitment_assessment_cron_job.rb delete mode 100644 app/jobs/commitment_assessment_job.rb delete mode 100644 app/models/commitment_status_deriver.rb delete mode 100644 app/models/criterion_assessor.rb create mode 100644 app/services/page_fetcher.rb delete mode 100644 lib/tasks/reevaluate.rake diff --git a/.env.example b/.env.example index 1d6a3ad..b7666c9 100644 --- a/.env.example +++ b/.env.example @@ -1 +1,5 @@ GEMINI_API_KEY= + +# Hosted defuddle worker (content extraction). URL defaults to the production worker. +DEFUDDLE_API_KEY= +DEFUDDLE_API_URL=https://deffudler.svc.canadasbuilding.com diff --git a/app/controllers/api/agent/commitment_matches_controller.rb b/app/controllers/api/agent/commitment_matches_controller.rb index 9be9502..7adc50b 100644 --- a/app/controllers/api/agent/commitment_matches_controller.rb +++ b/app/controllers/api/agent/commitment_matches_controller.rb @@ -8,10 +8,13 @@ def create matchable_id: params.require(:matchable_id), ) + # A match created by the agent has, by definition, already been reviewed. match.assign_attributes( relevance_score: params.require(:relevance_score), relevance_reasoning: params[:relevance_reasoning], matched_at: Time.current, + assessed: true, + assessed_at: Time.current, ) match.save! @@ -25,6 +28,7 @@ def create created: match.previously_new_record? } end + end end end diff --git a/app/controllers/api/agent/pages_controller.rb b/app/controllers/api/agent/pages_controller.rb index 9fbc98c..4f3eb9e 100644 --- a/app/controllers/api/agent/pages_controller.rb +++ b/app/controllers/api/agent/pages_controller.rb @@ -1,22 +1,24 @@ module Api module Agent class PagesController < BaseController - ALLOWED_SUFFIXES = %w[.canada.ca .gc.ca].freeze + ALLOWED_SUFFIXES = %w[.canada.ca .gc.ca .parl.ca].freeze def fetch url = params.require(:url) government_id = params.require(:government_id) unless government_url?(url) - render json: { error: "URL must be on a *.canada.ca or *.gc.ca domain" }, status: :unprocessable_entity + render json: { error: "URL must be on a *.canada.ca, *.gc.ca, or *.parl.ca domain" }, status: :unprocessable_entity return end # Fetch and parse the page - response = HTTP.timeout(connect: 5, read: 20) - .headers("User-Agent" => "BuildCanada-Tracker/1.0") - .follow(max_hops: 3) - .get(url) + begin + response = PageFetcher.get(url) + rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, SocketError => e + render json: { error: "Failed to fetch: #{e.class.name.demodulize} (#{e.message})" }, status: :bad_gateway + return + end unless response.status.success? render json: { error: "Failed to fetch: HTTP #{response.status}" }, status: :bad_gateway @@ -31,8 +33,13 @@ def fetch end # Parse HTML to markdown - prepared_html = Defuddle.prepare_html(response.body.to_s) - parsed_markdown, _parsed_html = Defuddle.defuddle(prepared_html) + begin + prepared_html = Defuddle.prepare_html(response.body.to_s) + parsed_markdown, _parsed_html = Defuddle.defuddle(prepared_html, url: final_url) + rescue Defuddle::ParseError => e + render json: { error: "Failed to parse page: #{e.message}" }, status: :bad_gateway + return + end # Extract title and date from meta tags doc = Nokogiri::HTML(response.body.to_s) diff --git a/app/jobs/agent_evaluate_commitment_job.rb b/app/jobs/agent_evaluate_commitment_job.rb index afe265d..5d9c9c0 100644 --- a/app/jobs/agent_evaluate_commitment_job.rb +++ b/app/jobs/agent_evaluate_commitment_job.rb @@ -59,7 +59,7 @@ def build_cmd(prompt, hook_script:) "--system-prompt", system_prompt, "--allowedTools", allowed_tools.join(","), "--permission-mode", "bypassPermissions", - "--model", ENV.fetch("AGENT_MODEL", "claude-sonnet-4-6"), + "--model", ENV.fetch("AGENT_MODEL", "claude-sonnet-5"), "--output-format", "text", "--settings", hook_settings ] @@ -91,7 +91,7 @@ def agent_env(commitment_id: nil, entry_id: nil) "CLAUDE_CODE_OAUTH_TOKEN" => ENV["CLAUDE_CODE_OAUTH_TOKEN"], "RAILS_API_URL" => ENV.fetch("RAILS_API_URL", "http://localhost:3000"), "RAILS_API_KEY" => Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"], - "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-sonnet-4-6"), + "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-sonnet-5"), "COMMITMENT_ID" => commitment_id&.to_s, "ENTRY_ID" => entry_id&.to_s, # Explicitly unset — subprocess must not access Rails credentials diff --git a/app/jobs/agent_process_entry_job.rb b/app/jobs/agent_process_entry_job.rb index 6ede719..86d0c86 100644 --- a/app/jobs/agent_process_entry_job.rb +++ b/app/jobs/agent_process_entry_job.rb @@ -48,7 +48,7 @@ def build_cmd(prompt, hook_script:) "--system-prompt", system_prompt, "--allowedTools", allowed_tools.join(","), "--permission-mode", "bypassPermissions", - "--model", ENV.fetch("AGENT_MODEL", "claude-opus-4-6"), + "--model", ENV.fetch("AGENT_MODEL", "claude-sonnet-5"), "--output-format", "text", "--settings", hook_settings ] @@ -79,7 +79,7 @@ def agent_env(commitment_id: nil, entry_id: nil) "CLAUDE_CODE_OAUTH_TOKEN" => ENV["CLAUDE_CODE_OAUTH_TOKEN"], "RAILS_API_URL" => ENV.fetch("RAILS_API_URL", "http://localhost:3000"), "RAILS_API_KEY" => Rails.application.credentials.dig(:agent, :api_key) || ENV["AGENT_API_KEY"], - "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-opus-4-6"), + "AGENT_MODEL" => ENV.fetch("AGENT_MODEL", "claude-sonnet-5"), "COMMITMENT_ID" => commitment_id&.to_s, "ENTRY_ID" => entry_id&.to_s, # Explicitly unset — subprocess must not access Rails credentials diff --git a/app/jobs/commitment_assessment_cron_job.rb b/app/jobs/commitment_assessment_cron_job.rb deleted file mode 100644 index ba994b3..0000000 --- a/app/jobs/commitment_assessment_cron_job.rb +++ /dev/null @@ -1,12 +0,0 @@ -class CommitmentAssessmentCronJob < ApplicationJob - queue_as :default - - def perform - commitment_ids = CommitmentMatch.unassessed.high_relevance - .select(:commitment_id).distinct.pluck(:commitment_id) - - Commitment.where(id: commitment_ids).find_each do |commitment| - CommitmentAssessmentJob.perform_later(commitment) - end - end -end diff --git a/app/jobs/commitment_assessment_job.rb b/app/jobs/commitment_assessment_job.rb deleted file mode 100644 index 799e4ad..0000000 --- a/app/jobs/commitment_assessment_job.rb +++ /dev/null @@ -1,103 +0,0 @@ -class CommitmentAssessmentJob < ApplicationJob - queue_as :default - - def perform(commitment) - if commitment.criteria.empty? - commitment.generate_criteria!(inline: true) - end - - unassessed_matches = commitment.commitment_matches.unassessed.high_relevance.includes(:matchable) - return if unassessed_matches.empty? - - evidence_items = unassessed_matches.map(&:matchable).compact - latest_evidence_date = evidence_items.filter_map { |m| evidence_date_for(m) }.max || Time.current - - commitment.criteria.find_each do |criterion| - assessor = CriterionAssessor.create!(record: criterion) - assessor.extract!(assessor.prompt(criterion, evidence_items)) - - new_status = assessor.assessment["new_status"] - next if new_status == criterion.status - - evidence_index = assessor.assessment["primary_evidence_index"] - primary_evidence = evidence_items[evidence_index] if evidence_index&.between?(0, evidence_items.size - 1) - primary_evidence ||= evidence_items.first - source = source_for(primary_evidence, commitment.government) - - CriterionAssessment.create!( - criterion: criterion, - previous_status: criterion.status, - new_status: new_status, - evidence_notes: assessor.assessment["evidence_notes"], - assessed_at: latest_evidence_date, - source: source - ) - - criterion.update!( - status: new_status, - evidence_notes: assessor.assessment["evidence_notes"], - assessed_at: latest_evidence_date - ) - end - - unassessed_matches.update_all(assessed: true, assessed_at: latest_evidence_date) - commitment.update!(last_assessed_at: latest_evidence_date) - end - - private - - def evidence_date_for(matchable) - case matchable - when Entry then matchable.published_at - when Bill then [ matchable.passed_house_first_reading_at, matchable.latest_activity_at ].compact.max - when StatcanDataset then matchable.last_synced_at - end - end - - def source_for(matchable, government) - return nil unless matchable - - case matchable - when Bill - Source.find_or_create_by!( - government: government, - source_type: :other, - source_type_other: "Parliamentary Bill", - title: "#{matchable.bill_number_formatted} — #{matchable.short_title}", - url: "https://www.parl.ca/legisinfo/en/bill/#{matchable.parliament_number}-#{matchable.data&.dig('SessionNumber')}/#{matchable.bill_number_formatted}", - date: evidence_date_for(matchable)&.to_date - ) - when Entry - source_type, source_type_other = source_type_for_feed(matchable.feed) - attrs = { - government: government, - source_type: source_type, - title: matchable.title, - url: matchable.url, - date: matchable.published_at&.to_date - } - attrs[:source_type_other] = source_type_other if source_type_other - Source.find_or_create_by!(attrs) - when StatcanDataset - Source.find_or_create_by!( - government: government, - source_type: :other, - source_type_other: "Statistics Canada Dataset", - title: matchable.name, - url: matchable.statcan_url, - date: matchable.last_synced_at&.to_date - ) - end - end - - def source_type_for_feed(feed) - title = feed.title.to_s.downcase - if title.include?("gazette") - [ :gazette_notice, nil ] - elsif title.include?("committee") - [ :committee_report, nil ] - else - [ :other, feed.title ] - end - end -end diff --git a/app/models/chat.rb b/app/models/chat.rb index dba2bc4..1d320de 100644 --- a/app/models/chat.rb +++ b/app/models/chat.rb @@ -3,6 +3,16 @@ class Chat < ApplicationRecord belongs_to :record, polymorphic: true, optional: true + class << self + # Rows from retired pipelines (CriterionAssessor, CommitmentStatusDeriver) remain as an + # audit log after their classes were removed. Load them as plain Chats instead of raising. + def find_sti_class(type_name) + super + rescue ActiveRecord::SubclassNotFound + self + end + end + def system_prompt end diff --git a/app/models/commitment_status_deriver.rb b/app/models/commitment_status_deriver.rb deleted file mode 100644 index 44d57d9..0000000 --- a/app/models/commitment_status_deriver.rb +++ /dev/null @@ -1,120 +0,0 @@ -class CommitmentStatusDeriver < Chat - include Structify::Model - - MODEL = "gemini-3.1-flash-lite-preview" - - after_create { with_model(MODEL, provider: :gemini, assume_exists: true) } - - schema_definition do - version 1 - name "CommitmentStatusDeriver" - description "Derives overall commitment status from criteria assessments" - field :derivation, :object, properties: { - "recommended_status" => { type: "string", enum: %w[not_started in_progress completed abandoned] }, - "reasoning" => { type: "string", description: "Explanation of why this status was recommended" }, - "confidence" => { type: "number", description: "0.0 to 1.0" } - } - end - - def system_prompt - "You are a government accountability analyst determining the overall status of a commitment based on its evaluation criteria." - end - - def prompt(commitment) - criteria_summary = commitment.criteria.where.not(category: :success).map do |c| - status_label = c.status - notes = c.evidence_notes.present? ? "\n Evidence: #{c.evidence_notes.truncate(300)}" : "" - " [#{c.category}] #{c.description}\n Status: #{status_label}#{notes}" - end.join("\n\n") - - <<~PROMPT - Determine the overall status of this government commitment based on its evaluation criteria. - - COMMITMENT: - Title: #{commitment.title} - Description: #{commitment.description} - Type: #{commitment.commitment_type} - Current Status: #{commitment.status} - - MATCHED EVIDENCE SUMMARY: - #{format_evidence_summary(commitment)} - - CRITERIA AND THEIR CURRENT ASSESSMENTS: - #{criteria_summary.presence || "(No criteria assessed yet)"} - - STATUS DEFINITIONS: - - not_started: No evidence of any action taken on this commitment - - in_progress: Some action has been taken but the commitment is not yet fulfilled - - completed: The government has done what it said it would do — completion criteria are met - - abandoned: The government has explicitly reversed or dropped this commitment - - EVIDENCE HIERARCHY (strictly enforced): - For "completed" status, one of these is REQUIRED: - - Legislative commitments: A matched bill with Royal Assent, OR a Gazette Part III entry - - Regulatory commitments: A matched Gazette Part II entry (enacted regulation) - - Spending/program commitments: Departmental news showing the program is operational - - For "in_progress" status, one of these is REQUIRED: - - A matched bill that has been introduced and is progressing (but no Royal Assent yet) - - A Gazette Part I entry (proposed regulation) - - Appropriation voted (matched to appropriation bill) with program evidence - - Departmental news showing concrete implementation steps - - Budget announcements alone (Budget 2025 text, budget speeches) are NOT sufficient evidence - for "in_progress" or "completed". A budget promise without a bill, regulation, or program - launch is "not_started". - - RULES: - - If completion criteria are met AND strong evidence exists per the hierarchy, recommend "completed". - - Progress criteria being met with appropriate evidence suggests "in_progress". - - If criteria are mostly "not_assessed", keep status as "not_started" unless there's clear evidence of action. - - If the only evidence is budget text or platform promises, recommend "not_started". - - Never recommend "abandoned" unless there is explicit evidence of reversal. - PROMPT - end - - private - - def format_evidence_summary(commitment) - matches = commitment.commitment_matches.includes(:matchable) - return "(No matched evidence)" if matches.empty? - - summary_parts = [] - - bill_matches = matches.select { |m| m.matchable_type == "Bill" } - if bill_matches.any? - bill_matches.each do |m| - bill = m.matchable - next unless bill - - royal_assent = bill.received_royal_assent_at.present? ? "Royal Assent #{bill.received_royal_assent_at.to_date}" : "No Royal Assent" - house_stage = if bill.passed_house_third_reading_at - "Passed House 3R" - elsif bill.passed_house_second_reading_at - "Passed House 2R" - elsif bill.passed_house_first_reading_at - "Passed House 1R" - else - "Introduced" - end - summary_parts << "BILL: #{bill.bill_number_formatted} — #{bill.short_title} (#{house_stage}, #{royal_assent})" - end - end - - entry_matches = matches.select { |m| m.matchable_type == "Entry" } - if entry_matches.any? - entry_matches.group_by { |m| m.matchable&.feed }.each do |feed, feed_matches| - next unless feed - - summary_parts << "#{feed.title}: #{feed_matches.size} matched entries" - end - end - - statcan_matches = matches.select { |m| m.matchable_type == "StatcanDataset" } - if statcan_matches.any? - summary_parts << "StatCan datasets: #{statcan_matches.size} matched" - end - - summary_parts.presence&.join("\n") || "(No matched evidence)" - end -end diff --git a/app/models/criterion_assessor.rb b/app/models/criterion_assessor.rb deleted file mode 100644 index a5b0a99..0000000 --- a/app/models/criterion_assessor.rb +++ /dev/null @@ -1,85 +0,0 @@ -class CriterionAssessor < Chat - include Structify::Model - - MODEL = "gemini-3.1-pro-preview" - - after_create { with_model(MODEL, provider: :gemini, assume_exists: true) } - - schema_definition do - version 1 - name "CriterionAssessor" - description "Assesses a criterion against matched evidence" - field :assessment, :object, properties: { - "new_status" => { type: "string", enum: %w[not_assessed met not_met no_longer_applicable] }, - "evidence_notes" => { type: "string", description: "Explanation referencing specific evidence" }, - "confidence" => { type: "number", description: "0.0 to 1.0" }, - "primary_evidence_index" => { type: "integer", description: "0-based index of the most relevant evidence item used for this assessment, from the MATCHED EVIDENCE list" } - } - end - - def system_prompt - "You are a government accountability analyst assessing specific criteria against available evidence." - end - - def prompt(criterion, evidence_items) - <<~PROMPT - You are assessing a specific criterion for a government commitment based on available evidence. - - COMMITMENT: - Title: #{criterion.commitment.title} - Description: #{criterion.commitment.description} - Type: #{criterion.commitment.commitment_type} - - CRITERION TO ASSESS: - Category: #{criterion.category} - Description: #{criterion.description} - Verification Method: #{criterion.verification_method} - Current Status: #{criterion.status} - Previous Evidence: #{criterion.evidence_notes} - - MATCHED EVIDENCE: - #{format_evidence(evidence_items)} - - ASSESSMENT RULES: - - met: Clear evidence that this criterion is fully satisfied - - not_met: Criterion is not satisfied, even if there is some progress toward it - - not_assessed: Insufficient evidence to make any determination - - no_longer_applicable: The commitment has been abandoned or the criterion is moot - - There is NO "partially met" status. Criteria are binary: met or not_met. - Progress toward meeting a criterion does not make it met — that is tracked separately by progress criteria. - - EVIDENCE STANDARDS (critical): - - Budget announcements, budget speeches, and platform promises are NOT evidence of action. - They are statements of intent. A budget saying "we will do X" does not mean X is done or in progress. - - For COMPLETION criteria: require Royal Assent (for legislation), Gazette Part II publication - (for regulations), or operational program evidence (for spending/programs). - - For PROGRESS criteria: require a bill progressing through Parliament, a Gazette Part I - proposed regulation, or departmental news showing concrete implementation steps. - - A bill that has NOT received Royal Assent means the legislation is NOT enacted. - The Budget Implementation Act (Bill C-15) being introduced or progressing is evidence of - progress, NOT completion. - - Be CONSERVATIVE. Only mark as "met" if evidence clearly supports it. - If current status is already "met" and no contradictory evidence, keep it "met". - Reference specific evidence items in your evidence_notes. - Set primary_evidence_index to the 0-based index of the evidence item that most directly supports your assessment. - PROMPT - end - - private - - def format_evidence(items) - items.each_with_index.map do |item, idx| - case item - when Entry - "[#{idx}] ENTRY [#{item.feed&.title}]: #{item.title} (#{item.published_at&.to_date})\n#{item.parsed_markdown&.truncate(1000)}" - when Bill - royal_assent = item.received_royal_assent_at.present? ? "ENACTED (Royal Assent #{item.received_royal_assent_at.to_date})" : "NOT ENACTED" - "[#{idx}] BILL: #{item.bill_number_formatted} - #{item.short_title}\nStatus: #{royal_assent}\nLatest: #{item.latest_activity} (#{item.latest_activity_at&.to_date})\nHouse: 1R=#{item.passed_house_first_reading_at&.to_date} 2R=#{item.passed_house_second_reading_at&.to_date} 3R=#{item.passed_house_third_reading_at&.to_date}\nSenate: 1R=#{item.passed_senate_first_reading_at&.to_date} 2R=#{item.passed_senate_second_reading_at&.to_date} 3R=#{item.passed_senate_third_reading_at&.to_date}" - when StatcanDataset - "[#{idx}] STATCAN: #{item.name}\nData: #{item.current_data&.first(3)&.to_json}" - end - end.join("\n\n---\n\n") - end -end diff --git a/app/models/entry.rb b/app/models/entry.rb index 54acbb9..a4fa6eb 100644 --- a/app/models/entry.rb +++ b/app/models/entry.rb @@ -45,7 +45,7 @@ def fetch_data!(inline: false) end # Fetch data from external source - r = HTTP.timeout(connect: 10, read: 30).get(url) + r = PageFetcher.get(url, connect: 10, read: 30) if r.status >= 300 or r.status < 200 Rails.logger.error("Error fetching data for entry #{id}: #{r.status}") @@ -53,7 +53,7 @@ def fetch_data!(inline: false) end self.raw_html = Defuddle.prepare_html(r.body.to_s) - self.parsed_markdown, self.parsed_html = Defuddle.defuddle(raw_html) + self.parsed_markdown, self.parsed_html = Defuddle.defuddle(raw_html, url: url) self.scraped_at = Time.now self.is_index = document_relative_links.any? diff --git a/app/services/page_fetcher.rb b/app/services/page_fetcher.rb new file mode 100644 index 0000000..a3d1622 --- /dev/null +++ b/app/services/page_fetcher.rb @@ -0,0 +1,16 @@ +# Fetches external pages with a consistent crawler identity. +# +# canada.ca's edge rejects bare bot-style and spoofed-browser User-Agents (connection reset +# or indefinite hang) but accepts an honest "compatible" crawler string with a contact URL. +module PageFetcher + USER_AGENT = "Mozilla/5.0 (compatible; BuildCanadaTracker/1.0; +https://buildcanada.com)".freeze + + class << self + def get(url, connect: 5, read: 20, max_hops: 3) + HTTP.timeout(connect: connect, read: read) + .headers("User-Agent" => USER_AGENT) + .follow(max_hops: max_hops) + .get(url) + end + end +end diff --git a/config/initializers/good_job.rb b/config/initializers/good_job.rb index 33fd919..d295a0c 100644 --- a/config/initializers/good_job.rb +++ b/config/initializers/good_job.rb @@ -35,12 +35,6 @@ description: "Generates evaluation criteria for commitments that don't have them yet", enabled_by_default: -> { Rails.env.production? } }, - commitment_assessment: { - cron: "0 */6 * * *", # Every 6 hours - class: "CommitmentAssessmentCronJob", - description: "Assess commitments with new evidence matches", - enabled_by_default: -> { Rails.env.production? } - }, target_date_extraction: { cron: "30 4 * * *", # Daily at 4:30 AM class: "TargetDateExtractionCronJob", diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 578e5e0..f3246bb 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -15,6 +15,8 @@ services: - SMTP_PASSWORD - SMTP_ADDRESS - GEMINI_API_KEY + - DEFUDDLE_API_KEY + - DEFUDDLE_API_URL - APPSIGNAL_PUSH_API_KEY - CLAUDE_CODE_OAUTH_TOKEN depends_on: @@ -42,6 +44,8 @@ services: - SMTP_PASSWORD - SMTP_ADDRESS - GEMINI_API_KEY + - DEFUDDLE_API_KEY + - DEFUDDLE_API_URL - APPSIGNAL_PUSH_API_KEY - CLAUDE_CODE_OAUTH_TOKEN restart: unless-stopped diff --git a/lib/defuddle.rb b/lib/defuddle.rb index d70fb93..6e03aa0 100644 --- a/lib/defuddle.rb +++ b/lib/defuddle.rb @@ -1,24 +1,45 @@ +require "http" +require "json" + +# Client for the hosted BuildCanada defuddle worker, which extracts the main content of a +# web page and returns it as markdown plus a cleaned HTML fragment. +# +# Configure with DEFUDDLE_API_KEY (required) and DEFUDDLE_API_URL (optional). module Defuddle - def self.defuddle(html) - temp_file = Tempfile.new("entry_html", encoding: "utf-8") - temp_file.write(html) + class ParseError < StandardError; end - md_json, err, status = Open3.capture3("defuddle", "parse", temp_file.path, "-m", "-j") - md_html, err, status = Open3.capture3("defuddle", "parse", temp_file.path, "-j") + DEFAULT_API_URL = "https://deffudler.svc.canadasbuilding.com".freeze - # replace anything before the first { deffudle returns errors and is dumb here. - md_json = "{" + md_json.split("{", 2).last - html_json = "{" + md_html.split("{", 2).last + class << self + # Sends already-fetched HTML to the worker and returns [markdown_content, html_content]. + # The page URL is passed along so relative links and site-specific rules resolve correctly. + def defuddle(html, url:) + response = HTTP.timeout(connect: 5, read: 60) + .headers("X-API-Key" => api_key, "Content-Type" => "application/json") + .post("#{api_url}/api/convert", json: { url: url, html: html }) - return JSON.parse(md_json)["content"], JSON.parse(html_json)["content"] - ensure - temp_file.close - temp_file.unlink - end + body = JSON.parse(response.body.to_s) + raise ParseError, "defuddle service returned HTTP #{response.status}: #{body["error"]}" unless response.status.success? + + [ body["content"], body["html"] ] + rescue HTTP::Error, JSON::ParserError => e + raise ParseError, "defuddle service request failed: #{e.message}" + end + + def prepare_html(html) + ic = Iconv.new("UTF-8//IGNORE", "UTF-8") + + ic.iconv(html + " ")[0..-2] + end + + private - def self.prepare_html(html) - ic = Iconv.new("UTF-8//IGNORE", "UTF-8") + def api_url + ENV.fetch("DEFUDDLE_API_URL", DEFAULT_API_URL).chomp("/") + end - ic.iconv(html + " ")[0..-2] + def api_key + ENV["DEFUDDLE_API_KEY"].presence || raise(ParseError, "DEFUDDLE_API_KEY is not set") + end end end diff --git a/lib/tasks/commitment_backfill.rake b/lib/tasks/commitment_backfill.rake index bade9a7..95c674a 100644 --- a/lib/tasks/commitment_backfill.rake +++ b/lib/tasks/commitment_backfill.rake @@ -42,19 +42,6 @@ namespace :commitments do puts "Enqueued relevance filtering for #{count} StatCan datasets" end - desc "Phase 5: Run initial assessment on all commitments with unassessed matches" - task assess: :environment do - count = 0 - commitment_ids = CommitmentMatch.unassessed.high_relevance - .select(:commitment_id).distinct.pluck(:commitment_id) - - Commitment.where(id: commitment_ids).find_each do |commitment| - CommitmentAssessmentJob.perform_later(commitment) - count += 1 - end - puts "Enqueued assessment for #{count} commitments" - end - desc "Run all backfill phases in sequence" task all: :environment do puts "=== Phase 1: Generate Criteria ===" @@ -64,8 +51,6 @@ namespace :commitments do puts " rake commitments:backfill:entries" puts " rake commitments:backfill:bills" puts " rake commitments:backfill:statcan" - puts "Then after those finish:" - puts " rake commitments:backfill:assess" end end end diff --git a/lib/tasks/matching_gaps.rake b/lib/tasks/matching_gaps.rake index 21a039d..747df8a 100644 --- a/lib/tasks/matching_gaps.rake +++ b/lib/tasks/matching_gaps.rake @@ -274,7 +274,6 @@ namespace :matching do puts "\n" + "=" * 70 puts "All gap-filling jobs enqueued." - puts "After jobs complete, run: rake matching:reassess" puts "=" * 70 end diff --git a/lib/tasks/reevaluate.rake b/lib/tasks/reevaluate.rake deleted file mode 100644 index c35696f..0000000 --- a/lib/tasks/reevaluate.rake +++ /dev/null @@ -1,117 +0,0 @@ -namespace :matching do - desc "Re-assess all commitments that have unassessed high-relevance matches" - task reassess: :environment do - commitment_ids = CommitmentMatch.unassessed.high_relevance - .select(:commitment_id).distinct.pluck(:commitment_id) - - puts "Found #{commitment_ids.size} commitments with unassessed high-relevance matches" - - count = 0 - Commitment.where(id: commitment_ids).find_each do |commitment| - CommitmentAssessmentJob.perform_later(commitment) - count += 1 - end - - puts "Enqueued assessment for #{count} commitments" - end - - desc "Report on commitment statuses and evidence quality" - task status_report: :environment do - puts "=" * 70 - puts "COMMITMENT STATUS REPORT" - puts "=" * 70 - - total = Commitment.count - by_status = Commitment.group(:status).count - puts "\nBy Status:" - by_status.each do |status, count| - pct = (count.to_f / total * 100).round(1) - puts " #{status}: #{count} (#{pct}%)" - end - - puts "\nEvidence Quality:" - - # Commitments with bill matches - with_bill = CommitmentMatch.where(matchable_type: "Bill") - .distinct.count(:commitment_id) - puts " With bill match: #{with_bill}" - - # With Royal Assent bills - royal_assent_bill_ids = Bill.where.not(received_royal_assent_at: nil).pluck(:id) - with_royal_assent = CommitmentMatch.where(matchable_type: "Bill", matchable_id: royal_assent_bill_ids) - .distinct.count(:commitment_id) - puts " With Royal Assent bill: #{with_royal_assent}" - - # With Gazette entries - gazette_ii_feed = Feed.find_by("title ILIKE ?", "%Gazette Part II%") - gazette_iii_feed = Feed.find_by("title ILIKE ?", "%Gazette Part III%") - - if gazette_ii_feed - g2_entry_ids = gazette_ii_feed.entries.pluck(:id) - with_g2 = CommitmentMatch.where(matchable_type: "Entry", matchable_id: g2_entry_ids) - .distinct.count(:commitment_id) - puts " With Gazette Part II match: #{with_g2}" - end - - if gazette_iii_feed - g3_entry_ids = gazette_iii_feed.entries.pluck(:id) - with_g3 = CommitmentMatch.where(matchable_type: "Entry", matchable_id: g3_entry_ids) - .distinct.count(:commitment_id) - puts " With Gazette Part III match: #{with_g3}" - end - - # With departmental news - dept_feeds = Feed.where("title ILIKE ANY(ARRAY[?])", [ - "%News Releases%", "%Press Releases%" - ]) - dept_entry_ids = Entry.where(feed: dept_feeds).pluck(:id) - with_dept = CommitmentMatch.where(matchable_type: "Entry", matchable_id: dept_entry_ids) - .distinct.count(:commitment_id) - puts " With departmental news match: #{with_dept}" - - # Criteria assessment coverage - puts "\nCriteria Assessment:" - total_criteria = Criterion.count - assessed = Criterion.where.not(status: :not_assessed).count - puts " Total criteria: #{total_criteria}" - puts " Assessed: #{assessed} (#{(assessed.to_f / total_criteria * 100).round(1)}%)" - puts " Unassessed: #{total_criteria - assessed}" - - # Evidence hierarchy violations - puts "\nEvidence Hierarchy Check:" - completed = Commitment.completed - completed.find_each do |c| - bill_matches = c.commitment_matches.where(matchable_type: "Bill") - entry_matches = c.commitment_matches.where(matchable_type: "Entry") - - has_royal_assent = bill_matches.any? do |m| - m.matchable.received_royal_assent_at.present? - end - - gazette_entry_ids = [] - gazette_entry_ids += g2_entry_ids if gazette_ii_feed - gazette_entry_ids += g3_entry_ids if gazette_iii_feed - - has_gazette = entry_matches.where(matchable_id: gazette_entry_ids).exists? if gazette_entry_ids.any? - - has_dept_news = entry_matches.where(matchable_id: dept_entry_ids).exists? if dept_entry_ids.any? - - unless has_royal_assent || has_gazette || has_dept_news - puts " WARNING: Commitment ##{c.id} '#{c.title.truncate(60)}' marked completed without strong evidence" - end - end - - puts "=" * 70 - end - - desc "Full pipeline: fix gaps, reassess, rederive (run sequentially as jobs complete)" - task full_pipeline: :environment do - puts "Step 1: Fixing matching gaps..." - Rake::Task["matching:all_gaps"].invoke - - puts "\nJobs are enqueued. The full pipeline is:" - puts " 1. rake matching:all_gaps (done — jobs running)" - puts " 2. rake matching:reassess (run after gap jobs finish)" - puts " 3. rake matching:status_report (run after assessment jobs finish)" - end -end From fbb954ed8efd12b9ef866a298bf8c63affd8bfd4 Mon Sep 17 00:00:00 2001 From: xrendan Date: Thu, 10 Sep 2026 11:40:39 -0600 Subject: [PATCH 2/2] Fix CI: lint offense, stale test fixtures, StatCan 404, Brakeman version pin - Remove an extra blank line RuboCop flagged in the agent matches controller - Update the relevance filter test to use the `broken` status; `abandoned` was renamed some time ago and the test has failed since - StatcanDatasetsController#show now finds by name or id and returns 404 when nothing matches, instead of rendering `null` with 200 - Drop `--ensure-latest` from bin/brakeman. It exits 5 whenever a newer Brakeman ships, which is why scan_ruby has failed on main since March - Only render entry URLs as links in the scraping health admin view when they are http(s); add config/brakeman.ignore for that check (Brakeman cannot see the guard) and for the Rails 8.0 end-of-life notice --- .../agent/commitment_matches_controller.rb | 1 - .../statcan_datasets_controller.rb | 9 ++- app/views/avo/scraping_health/index.html.erb | 6 +- bin/brakeman | 2 - config/brakeman.ignore | 59 +++++++++++++++++++ .../commitment_relevance_filter_job_test.rb | 8 +-- 6 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 config/brakeman.ignore diff --git a/app/controllers/api/agent/commitment_matches_controller.rb b/app/controllers/api/agent/commitment_matches_controller.rb index 7adc50b..625d656 100644 --- a/app/controllers/api/agent/commitment_matches_controller.rb +++ b/app/controllers/api/agent/commitment_matches_controller.rb @@ -28,7 +28,6 @@ def create created: match.previously_new_record? } end - end end end diff --git a/app/controllers/statcan_datasets_controller.rb b/app/controllers/statcan_datasets_controller.rb index 4634e42..a1a6d47 100644 --- a/app/controllers/statcan_datasets_controller.rb +++ b/app/controllers/statcan_datasets_controller.rb @@ -1,6 +1,11 @@ class StatcanDatasetsController < ApplicationController def show - dataset = StatcanDataset.find_by(name: params[:id]) - render json: dataset + dataset = StatcanDataset.find_by(name: params[:id]) || StatcanDataset.find_by(id: params[:id]) + + if dataset + render json: dataset + else + render json: { error: "Dataset not found" }, status: :not_found + end end end diff --git a/app/views/avo/scraping_health/index.html.erb b/app/views/avo/scraping_health/index.html.erb index efc895b..ac3b61a 100644 --- a/app/views/avo/scraping_health/index.html.erb +++ b/app/views/avo/scraping_health/index.html.erb @@ -98,7 +98,11 @@ <%= entry.published_at ? time_ago_in_words(entry.published_at) + " ago" : "—" %> - <%= link_to entry.url, entry.url, target: "_blank", class: "text-blue-600 hover:underline", title: entry.url %> + <% if entry.url.to_s.match?(%r{\Ahttps?://}) %> + <%= link_to entry.url, entry.url, target: "_blank", class: "text-blue-600 hover:underline", title: entry.url %> + <% else %> + <%= entry.url %> + <% end %> <% end %> diff --git a/bin/brakeman b/bin/brakeman index ace1c9b..171ac12 100755 --- a/bin/brakeman +++ b/bin/brakeman @@ -2,6 +2,4 @@ require "rubygems" require "bundler/setup" -ARGV.unshift("--ensure-latest") - load Gem.bin_path("brakeman", "brakeman") diff --git a/config/brakeman.ignore b/config/brakeman.ignore new file mode 100644 index 0000000..ebfcdd0 --- /dev/null +++ b/config/brakeman.ignore @@ -0,0 +1,59 @@ +{ + "ignored_warnings": [ + { + "warning_type": "Cross-Site Scripting", + "warning_code": 4, + "fingerprint": "2aa3ad93f74ccdf22492d14b0d070e303e4a218aa07944e62fcc798e97d68a47", + "check_name": "LinkToHref", + "message": "Potentially unsafe model attribute in `link_to` href", + "file": "app/views/avo/scraping_health/index.html.erb", + "line": 102, + "link": "https://brakemanscanner.org/docs/warning_types/link_to_href", + "code": "link_to((Unresolved Model).new.url, (Unresolved Model).new.url, :target => \"_blank\", :class => \"text-blue-600 hover:underline\", :title => (Unresolved Model).new.url)", + "render_path": [ + { + "type": "controller", + "class": "Avo::ScrapingHealthController", + "method": "index", + "line": 23, + "file": "app/controllers/avo/scraping_health_controller.rb", + "rendered": { + "name": "avo/scraping_health/index", + "file": "app/views/avo/scraping_health/index.html.erb" + } + } + ], + "location": { + "type": "template", + "template": "avo/scraping_health/index" + }, + "user_input": "(Unresolved Model).new.url", + "confidence": "Weak", + "cwe_id": [ + 79 + ], + "note": "entry.url is rendered as a link only when it starts with http:// or https:// (see the guard in the view); Brakeman cannot see the conditional." + }, + { + "warning_type": "Unmaintained Dependency", + "warning_code": 122, + "fingerprint": "98b26f60d776fd41ee6f088c833725145be9aac2d7c5b33780241c273622db42", + "check_name": "EOLRails", + "message": "Support for Rails 8.0.2 ends on 2026-10-07", + "file": "Gemfile.lock", + "line": 331, + "link": "https://brakemanscanner.org/docs/warning_types/unmaintained_dependency/", + "code": null, + "render_path": null, + "location": null, + "user_input": null, + "confidence": "Medium", + "cwe_id": [ + 1104 + ], + "note": "Rails 8.0.x support ends 2026-10-07; the Rails upgrade is tracked separately from this scan." + } + ], + "updated": "2026-09-10", + "brakeman_version": "8.0.4" +} diff --git a/test/jobs/commitment_relevance_filter_job_test.rb b/test/jobs/commitment_relevance_filter_job_test.rb index 06dfff0..fb1713e 100644 --- a/test/jobs/commitment_relevance_filter_job_test.rb +++ b/test/jobs/commitment_relevance_filter_job_test.rb @@ -108,13 +108,13 @@ class CommitmentRelevanceFilterJobTest < ActiveJob::TestCase assert_not_includes result.to_a, commitment end - test "excludes abandoned commitments regardless of date" do + test "excludes broken commitments regardless of date" do commitment = Commitment.create!( government: @government, - title: "Abandoned commitment", - description: "This was abandoned", + title: "Broken commitment", + description: "This was broken", commitment_type: :spending, - status: :abandoned, + status: :broken, date_promised: Date.new(2025, 1, 1) )