From c88b6d9ea8a1f9dde647abd1516f50f9d06b2d73 Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 21 Jan 2019 17:46:55 +0530 Subject: [PATCH 01/11] [deps] Bundle update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 709c1732..a4144903 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -58,4 +58,4 @@ DEPENDENCIES thin BUNDLED WITH - 2.2.6 + 2.2.7 From fab37774bad8c0e321fc8d6f39f9e4ad4f6fca12 Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 21 Jan 2019 18:07:17 +0530 Subject: [PATCH 02/11] Adds a new IFSC.html page - Adds a PINCODE attribute as well --- app.rb | 28 ++++++++++++++++++++- views/ifsc.erb | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 views/ifsc.erb diff --git a/app.rb b/app.rb index 93e4075a..d97a74f7 100644 --- a/app.rb +++ b/app.rb @@ -80,12 +80,27 @@ def ifsc_data(code) data = settings.redis.hgetall code if !data.empty? + + encoding_options = { + :invalid => :replace, # Replace invalid byte sequences + :undef => :replace, # Replace anything not defined in ASCII + :replace => '', # Use a blank for those replacements + :universal_newline => true # Always break lines with \n + } + + data['PINCODE'] = nil + data['ADDRESS'] = data['ADDRESS'].encode(Encoding.find('ASCII'), encoding_options) data['BANK'], data['BANKCODE'] = bank_details(code) data['IFSC'] = code data['RTGS'] = strtobool data['RTGS'] data['NEFT'] = strtobool data['NEFT'] data['IMPS'] = strtobool data['IMPS'] data['UPI'] = strtobool data['UPI'] + + if data['ADDRESS'] =~ /(\d{6})/ + data['PINCODE'] = data['ADDRESS'].match /(\d{6})/ + end + settings.metrics.increment code else data = nil @@ -96,13 +111,24 @@ def ifsc_data(code) get '/' do readme = File.read 'README.md' - erb :index, locals: { text: markdown(readme) } + erb :index end get '/metrics' do settings.metrics.format end +get '/:code.html' do + begin + data = ifsc_data(params['code']) + erb :ifsc, locals: { data: data } + rescue StandardError => e + puts e + status 404 + json 'Not Found' + end +end + get '/:code' do begin data = ifsc_data(params['code']) diff --git a/views/ifsc.erb b/views/ifsc.erb new file mode 100644 index 00000000..c372f548 --- /dev/null +++ b/views/ifsc.erb @@ -0,0 +1,66 @@ + + + + + Razorpay IFSC Toolkit + + + + + + +
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IFSC<%= data['IFSC']%>
Address<%= data['ADDRESS']%>
Contact<%= data['CONTACT']%>
City<%= data['CITY']%>
District<%= data['DISTRICT']%>
State<%= data['STATE']%>
RTGS Supported<%= data['RTGS'] ? "Yes" : "No" %>
Bank<%= data['BANK']%> (<%=data['BANKCODE']%>)
+
+ + From 0438063bd83111fe34c80ca9e515917dacc9bdf2 Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 21 Jan 2019 18:17:50 +0530 Subject: [PATCH 03/11] Adds support for long paths with Address --- app.rb | 14 +++++++++++++- views/ifsc.erb | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/app.rb b/app.rb index d97a74f7..29dc2063 100644 --- a/app.rb +++ b/app.rb @@ -5,6 +5,7 @@ require 'ifsc' require './metrics' require 'sinatra/json' +require "sinatra/multi_route" require 'secure_headers' class IFSCPlus < Razorpay::IFSC::IFSC @@ -118,7 +119,18 @@ def ifsc_data(code) settings.metrics.format end -get '/:code.html' do +get '/:code.html', %r{/\w+/\w+/(?[A-Z0-9]{11})} do + begin + data = ifsc_data(params['code']) + erb :ifsc, locals: { data: data } + rescue StandardError => e + puts e + status 404 + json 'Not Found' + end +end + +get '/.html' do begin data = ifsc_data(params['code']) erb :ifsc, locals: { data: data } diff --git a/views/ifsc.erb b/views/ifsc.erb index c372f548..be549026 100644 --- a/views/ifsc.erb +++ b/views/ifsc.erb @@ -53,7 +53,7 @@ <%= data['STATE']%> - RTGS Supported + RTGS <%= data['RTGS'] ? "Yes" : "No" %> From 46c149012cd2e1693a659e1168a09d09d346cb36 Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 21 Jan 2019 19:02:12 +0530 Subject: [PATCH 04/11] Adds sitemap --- .gitignore | 3 ++- init.rb | 47 +++++++++++++++++++++++++++++++++++++---------- public/robots.txt | 1 + 3 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 public/robots.txt diff --git a/.gitignore b/.gitignore index 5657f6ea..44911cf4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -vendor \ No newline at end of file +vendor +public/sitemap*.xml* diff --git a/init.rb b/init.rb index b39e1cf5..786045b7 100644 --- a/init.rb +++ b/init.rb @@ -1,6 +1,8 @@ require 'json' require 'redis' require 'benchmark' +require 'to_slug' +require 'sitemap_generator' redis = Redis.new @@ -8,24 +10,49 @@ def log(msg) puts "[+] (#{Time.now.strftime('%r')}) #{msg}" end +def add_to_sitemap(sitemap, ifsc, data) + url = nil + if data['BANK'] && data['BRANCH'] + url = "/#{data['BANK'].to_s.to_slug}/#{data['BRANCH'].to_s.to_slug}/#{ifsc}" + end + if url + config = { lastmod: Time.now, changefreq: 'monthly', priority: 0.3 } + sitemap.add url, config + end +end + Benchmark.bm(18) do |bm| bm.report('Ingest:') do - Dir.glob('data/*.json') do |file| - log "Processing #{file}" - bank = File.basename file, '.json' - if Regexp.new('[A-Z]{4}').match(bank) - data = JSON.parse File.read file - data.each do |ifsc, d| - # Remove the extra keys from the JSON files - d.delete_if { |key| %w[BANK IFSC].include? key } - redis.hmset ifsc, *d + SitemapGenerator::Sitemap.default_host = 'https://ifsc.razorpay.com' + SitemapGenerator::Sitemap.create_index = true + + SitemapGenerator::Sitemap.create do |s| + s.add '/', changefreq: 'daily', priority: 0.9 + Dir.glob('data/*.json') do |file| + log "Processing #{file}" + bank = File.basename file, '.json' + if Regexp.new('[A-Z]{4}').match(bank) + data = JSON.parse File.read file + data.each do |ifsc, d| + begin + add_to_sitemap(s, ifsc, d) + rescue Exception => e + puts e + end + + # Remove the extra keys from the JSON files + d.delete_if { |key| %w[BANK IFSC].include? key } + redis.hmset ifsc, *d + end + log "Processed #{data.size} entries" end - log "Processed #{data.size} entries" end end end + bm.report('Dump:') do redis.save end end + log('Data saved to Redis') diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 00000000..8d534da8 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1 @@ +Sitemap: https://ifsc.razorpay.com/sitemap.xml.gz From 0327fe6f113346de916b84bd5a8acb1c27a4b255 Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 21 Jan 2019 19:04:57 +0530 Subject: [PATCH 05/11] [docker] Update bundler in Docker --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index fbd74d06..d699b246 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ COPY data /app/data/ RUN echo "** installing deps **" && \ apk --no-cache add redis && \ echo "** installing ruby gems **" && \ + gem install bundler && \ + echo "** updating bundler **" && \ bundle install && \ echo "** starting redis-server **" && \ redis-server --daemonize yes && \ From 733f3699f4cdfcb9d75dca1c1b9423dc6313b524 Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 21 Jan 2019 19:08:28 +0530 Subject: [PATCH 06/11] [deps] Move CI deps to Gemfile.build --- Gemfile.build | 4 +++- Gemfile.build.lock | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Gemfile.build b/Gemfile.build index 20104143..a39585f9 100644 --- a/Gemfile.build +++ b/Gemfile.build @@ -1,4 +1,6 @@ # Gemfile used in the build docker container source "https://rubygems.org" -gem "redis" \ No newline at end of file +gem "redis" +gem "sitemap_generator" +gem 'to_slug', '~> 1.0', '>= 1.0.8' diff --git a/Gemfile.build.lock b/Gemfile.build.lock index 138c8052..193b15d3 100644 --- a/Gemfile.build.lock +++ b/Gemfile.build.lock @@ -1,13 +1,19 @@ GEM remote: https://rubygems.org/ specs: - redis (4.1.3) + builder (3.2.4) + redis (4.2.5) + sitemap_generator (6.1.2) + builder (~> 3.0) + to_slug (1.0.8) PLATFORMS ruby DEPENDENCIES redis + sitemap_generator + to_slug (~> 1.0, >= 1.0.8) BUNDLED WITH - 2.0.2 + 2.2.7 From dc5ed492993fa81d1500956e3b239ae8cfd7b049 Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 21 Jan 2019 19:25:44 +0530 Subject: [PATCH 07/11] [sitemap] Serve sitemap on the stage website --- init.rb | 3 ++- public/robots.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/init.rb b/init.rb index 786045b7..2ff0b927 100644 --- a/init.rb +++ b/init.rb @@ -23,7 +23,8 @@ def add_to_sitemap(sitemap, ifsc, data) Benchmark.bm(18) do |bm| bm.report('Ingest:') do - SitemapGenerator::Sitemap.default_host = 'https://ifsc.razorpay.com' + # CHANGE TO .com before merging + SitemapGenerator::Sitemap.default_host = 'https://ifsc.stage.razorpay.in' SitemapGenerator::Sitemap.create_index = true SitemapGenerator::Sitemap.create do |s| diff --git a/public/robots.txt b/public/robots.txt index 8d534da8..4b3c0d9e 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1 +1 @@ -Sitemap: https://ifsc.razorpay.com/sitemap.xml.gz +Sitemap: https://ifsc.stage.razorpay.in/sitemap.xml.gz From b09e858247c68eb5d14d161cb6187d81a2961ca2 Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 21 Jan 2019 20:21:08 +0530 Subject: [PATCH 08/11] Remove extra route --- app.rb | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/app.rb b/app.rb index 29dc2063..fd721ac5 100644 --- a/app.rb +++ b/app.rb @@ -111,7 +111,6 @@ def ifsc_data(code) end get '/' do - readme = File.read 'README.md' erb :index end @@ -130,17 +129,6 @@ def ifsc_data(code) end end -get '/.html' do - begin - data = ifsc_data(params['code']) - erb :ifsc, locals: { data: data } - rescue StandardError => e - puts e - status 404 - json 'Not Found' - end -end - get '/:code' do begin data = ifsc_data(params['code']) From 89c09e097f2cb896eeebaeaff7e38a36e71bf8d7 Mon Sep 17 00:00:00 2001 From: Nemo Date: Mon, 21 Jan 2019 20:40:09 +0530 Subject: [PATCH 09/11] Allow dashes in address/bank names --- app.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.rb b/app.rb index fd721ac5..48065533 100644 --- a/app.rb +++ b/app.rb @@ -118,7 +118,7 @@ def ifsc_data(code) settings.metrics.format end -get '/:code.html', %r{/\w+/\w+/(?[A-Z0-9]{11})} do +get '/:code.html', %r{/[\w-]+/[\w-]+/(?[A-Z0-9]{11})} do begin data = ifsc_data(params['code']) erb :ifsc, locals: { data: data } From 93657410429666d5e366a86876859c7fa263bbde Mon Sep 17 00:00:00 2001 From: Nemo Date: Wed, 27 Feb 2019 17:21:30 +0530 Subject: [PATCH 10/11] Adds /banks/:bank API --- app.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app.rb b/app.rb index 48065533..33971591 100644 --- a/app.rb +++ b/app.rb @@ -54,6 +54,9 @@ def get_bank_code(code) end set :redis, Redis.new set :server, 'thin' + set :bank_names, JSON.parse(File.read('data/banknames.json')) + set :sublet_list, JSON.parse(File.read('data/sublet.json')) + set :banks_list, JSON.parse(File.read('data/banks.json')) set :metrics, Metrics.new end @@ -118,6 +121,7 @@ def ifsc_data(code) settings.metrics.format end + get '/:code.html', %r{/[\w-]+/[\w-]+/(?[A-Z0-9]{11})} do begin data = ifsc_data(params['code']) @@ -129,6 +133,19 @@ def ifsc_data(code) end end +get '/banks/:code' do + begin + bank = settings.banks_list[params['code']] + bank['bank_code'] = bank['micr'][3..5] + bank['name'] = settings.bank_names[params['code']] + return json bank + rescue Exception => e + puts e + status 404 + json 'Not Found' + end +end + get '/:code' do begin data = ifsc_data(params['code']) From 9b565ff6d803ad4edf876cc3bec9ca18df81e0c8 Mon Sep 17 00:00:00 2001 From: Nemo Date: Thu, 28 Jan 2021 15:01:11 +0530 Subject: [PATCH 11/11] Removes extra JSON files --- app.rb | 20 ++------------------ entrypoint.sh | 2 +- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/app.rb b/app.rb index 33971591..f9e1a3b2 100644 --- a/app.rb +++ b/app.rb @@ -54,9 +54,6 @@ def get_bank_code(code) end set :redis, Redis.new set :server, 'thin' - set :bank_names, JSON.parse(File.read('data/banknames.json')) - set :sublet_list, JSON.parse(File.read('data/sublet.json')) - set :banks_list, JSON.parse(File.read('data/banks.json')) set :metrics, Metrics.new end @@ -85,15 +82,7 @@ def ifsc_data(code) if !data.empty? - encoding_options = { - :invalid => :replace, # Replace invalid byte sequences - :undef => :replace, # Replace anything not defined in ASCII - :replace => '', # Use a blank for those replacements - :universal_newline => true # Always break lines with \n - } - data['PINCODE'] = nil - data['ADDRESS'] = data['ADDRESS'].encode(Encoding.find('ASCII'), encoding_options) data['BANK'], data['BANKCODE'] = bank_details(code) data['IFSC'] = code data['RTGS'] = strtobool data['RTGS'] @@ -127,7 +116,6 @@ def ifsc_data(code) data = ifsc_data(params['code']) erb :ifsc, locals: { data: data } rescue StandardError => e - puts e status 404 json 'Not Found' end @@ -135,12 +123,8 @@ def ifsc_data(code) get '/banks/:code' do begin - bank = settings.banks_list[params['code']] - bank['bank_code'] = bank['micr'][3..5] - bank['name'] = settings.bank_names[params['code']] - return json bank + return json IFSCPlus::Bank.get_details params['code'] rescue Exception => e - puts e status 404 json 'Not Found' end @@ -158,6 +142,6 @@ def ifsc_data(code) rescue StandardError => e puts e status 404 - json 'Not Found' + json "Not Found" end end diff --git a/entrypoint.sh b/entrypoint.sh index 7a766ed8..0886bdfe 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,4 +1,4 @@ #!/usr/bin/dumb-init /bin/sh cd /app redis-server --daemonize yes -bundle exec rackup --host 0.0.0.0 --port 3000 \ No newline at end of file +bundle exec rackup --host 0.0.0.0 --port 3000