AO3-6978 Create and fill new table for import urls

db/migrate/20260318013837_create_imported_urls.rb

@@ -7,10 +7,16 @@ def change
       t.string :minimal, null: false
       t.string :minimal_no_protocol_no_www, null: false
       t.string :no_www, null: false
-      t.string :with_www, null: false
+      # as of this migration, with_www is about 4 chars longer than the original string
+      # since the default string length is limited to 255, we need at least 259
+      # putting 300 to give a bit of extra wiggle room
+      t.string :with_www, null: false, limit: 300
       t.string :with_http, null: false
       t.string :with_https, null: false
-      t.string :encoded, null: false
+      # encoded can be much larger than original if there's enough special characters in the string
+      # setting a limit of 2080 to match the limit used on abuse reports
+      # as that is the largest url limit currently on the project
+      t.string :encoded, null: false, limit: 2080
       t.string :decoded, null: false
 
       t.timestamps

lib/tasks/work_import_urls.rake

@@ -7,17 +7,24 @@ namespace :work_import_urls do
     scope = Work.where("imported_from_url IS NOT NULL AND imported_from_url != ''")
     total = scope.count
     processed = 0
+    failed = 0
 
     puts "Backfilling #{total} work_import_urls records..."
 
     scope.find_each(batch_size: batch_size) do |work|
       next if ImportedUrl.exists?(work_id: work.id)
 
-      work.imported_url = ImportedUrl.create(original: work.imported_from_url)
+      begin
+        work.imported_url = ImportedUrl.create(original: work.imported_from_url)
+      rescue StandardError => e
+        puts "------- Error on import for work_id #{work.id}: #{e.inspect}"
+        failed += 1
+        next
+      end
       processed += 1
       puts "Processed #{processed}/#{total}" if (processed % batch_size).zero?
     end
 
-    puts "Done! Backfilled #{processed} records."
+    puts "Done! Backfilled #{processed} records, failed #{failed} records."
   end
 end

spec/models/imported_url_spec.rb

@@ -17,4 +17,17 @@
     expect(url.encoded).to eq("http://www.trickster.org/llwyden/misc/cracked.html")
     expect(url.decoded).to eq("http://www.trickster.org/llwyden/misc/cracked.html")
   end
+
+  it "handles longer urls where the encoded column will be much larger gracefully" do
+    # default string column max is 255 so we create an url with exactly 255 chars
+    # and encoding that will extend the
+    repeated_string = "#() {}" * 38
+    long_url = "http://www.faikes.org/#{repeated_string}.html"
+
+    formatter = UrlFormatter.new(long_url)
+    url.original = formatter.original
+
+    expect { url.save }
+      .not_to raise_error
+  end
 end