This is how we created news.dpi.dev, a feed of all DPI trainees blog posts on dev.to. We used sidekiq for background processing, but you could easily use another gem like good_job.
app/controllers/news_controller.rb
class NewsController < ApplicationController
layout "news"
skip_before_action :authenticate_user!
before_action { authorize(:news) }
def index
@q = policy_scope(DevtoArticle).page(params[:page]).ransack(params[:q])
@articles = @q.result.includes(:author).default_order
end
end
app/models/devto_article.rb
# == Schema Information
#
# Table name: devto_articles
#
# id :uuid not null, primary key
# description :text
# published_at :datetime
# title :string
# url :string
# created_at :datetime not null
# updated_at :datetime not null
# author_id :uuid not null
# devto_id :integer
#
# Indexes
#
# index_devto_articles_on_author_id (author_id)
#
# Foreign Keys
#
# fk_rails_... (author_id => users.id)
#
class DevtoArticle < ApplicationRecord
include Ransackable
belongs_to :author, class_name: "User", foreign_key: "author_id"
scope :default_order, -> { order(published_at: :desc) }
end
app/models/user.rb
class User < ApplicationRecord
...
include Blogable
...
end
app/models/concerns/user/blogable/rb
module User::Blogable
extend ActiveSupport::Concern
included do
scope :bloggers, -> { where.not(devto_username: nil)}
has_many :devto_articles, class_name: "DevtoArticle", foreign_key: "author_id"
before_save { self.devto_username = devto_username.downcase if devto_username.present? }
end
class_methods do
def fetch_devto_articles
User.bloggers.each do |u|
u.fetch_devto_articles
sleep(1) # hack fix for 429 status code
end
end
end
def fetch_devto_articles
return unless devto_username.present?
DevtoService.fetch_articles({username: devto_username})&.each do |article|
devto_article = devto_articles.find_or_initialize_by(devto_id: article["id"])
devto_article.title = article["title"]
devto_article.description = article["description"]
devto_article.url = article["url"]
devto_article.published_at = article["published_timestamp"]
devto_article.save
end
end
end
app/services/devto_service.rb
require 'http'
require 'json'
class DevtoService
BASE_URL = "https://dev.to/api"
def self.fetch_articles(params)
articles_url = "#{BASE_URL}/articles?#{params.to_query}"
response = HTTP.headers('Accept': 'application/vnd.forem.api-v1+json').get(articles_url)
if response.status.success?
body = response.body.to_s
parsed_response = JSON.parse(body)
return parsed_response
else
puts "Failed to fetch articles for username #{params.dig(:username)}. Response code: #{response.status}"
return nil
end
end
end
Setting up the cron job will be different depending on which background processor you choose to use (sidekiq, goodjob, etc.)
config/schedule.yml
fetch_devto_articles:
cron: "*/30 * * * *" # execute every 30 minutes
class: "FetchDevtoArticlesJob"
queue: news
app/jobs/fetch_devto_articles_job.rb
class FetchDevtoArticlesJob < ApplicationJob
queue_as :news
def perform
User.fetch_devto_articles
end
end