Created
November 27, 2016 13:46
-
-
Save Azdaroth/17ee688d9b1098f5bfa8bfeeed0c4479 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class NevacheImport | |
attr_reader :account, :base_url, :mechanize, :csv_dir, :rentals_csv_file, | |
:rentals_details_csv_file, :addresses_csv_file | |
def initialize(account:, csv_dir:) | |
@account = account | |
@base_url = "http://www.nevache-tourisme.fr/fr/locations-meubles/" | |
@mechanize = Mechanize.new | |
@csv_dir = csv_dir | |
@rentals_csv_file = File.join(csv_dir, "a_structures.csv") | |
@rentals_details_csv_file = File.join(csv_dir, "a_structures_detail.csv") | |
@addresses_csv_file = File.join(csv_dir, "a_adresses.csv") | |
end | |
def import! | |
Importer.new.import!(account, full_mapping) | |
end | |
def main_page | |
@main_page ||= mechanize.get(base_url) | |
end | |
def rentals_urls | |
@rentals_urls ||= begin | |
main_page.links_with(class: "structure-resume").map(&:href).map do |relative_url| | |
"#{base_url}#{relative_url}" | |
end | |
end | |
end | |
def rentals_mappings_from_page | |
@rentals_mappings_from_page ||= begin | |
rentals_urls.map do |url| | |
RentalPage.new(mechanize, url).rental_mapping | |
end | |
end | |
end | |
def full_mapping | |
rentals_mappings_from_page.map do |mapping| | |
mapping[:rental_attributes] = attributes_for_rental_from_mapping(mapping) | |
mapping[:rentals_contact_attributes] = attributes_for_rental_contact_from_mapping(mapping) | |
mapping[:rentals_amenities_keys] = rentals_amenities_keys(mapping) | |
mapping | |
end | |
end | |
def attributes_for_rental_from_mapping(mapping) | |
csv_row = rentals_from_csv.find { |r| r[:id].to_s == mapping[:id].to_s } || {} | |
details = rentals_details_from_csv.find { |r| r[:id_structure].to_s == csv_row[:id].to_s } || {} | |
address = addresses_from_csv.find { |r| r[:id].to_s == csv_row[:id_adresse].to_s } || {} | |
attributes = {} | |
attributes[:name] = csv_row[:libelle] | |
attributes[:zip] = csv_row[:cp] | |
attributes[:city] = csv_row[:commune] | |
attributes[:address1] = address[:libelle] | |
attributes[:description_fr] = strip(csv_row[:description]) | |
attributes[:summary_fr] = strip(csv_row[:resume]).truncate(150) | |
attributes[:surface] = (details[:surface].presence && details[:surface].to_i) | |
attributes[:notes] = strip(details[:commentaire]) | |
attributes[:surface_unit] = "metric" if attributes[:surface].present? | |
attributes[:bedrooms_count] = details[:nb_chambre].to_i | |
attributes[:currency] = "EUR" | |
attributes[:country_code] = "FR" | |
attributes | |
end | |
def attributes_for_rental_contact_from_mapping(mapping) | |
csv_row = rentals_from_csv.find { |r| r[:id].to_s == mapping[:id].to_s } or return {} | |
attributes = {} | |
attributes[:fullname] = csv_row[:email] | |
attributes[:email] = csv_row[:email] | |
attributes[:phones] = [csv_row[:tel], csv_row[:mobile]].compact.join(", ") | |
attributes[:spoken_languages] = ["FR"] | |
attributes | |
end | |
def rentals_amenities_keys(mapping) | |
details_amenities_mapping = { | |
animal: "pets_considered", | |
balcon_terrasse: "balcony", | |
jardin: "garden", | |
telephone: "telephone", | |
sauna_jacuzzi_bain: "jacuzzi", | |
acces_handicape: "handicap_accessible", | |
acces_voiture: "car_included", | |
cuisine: "full_equipped_kitchen", | |
parking: "parking_included", | |
tele: "tv", | |
wifi: "internet", | |
lave_linge: "washing_machine", | |
lave_vaisselle: "dishwasher" | |
} | |
keys = [] | |
details = rentals_details_from_csv.find { |r| r[:id_structure].to_s == mapping[:id].to_s } || {} | |
details_amenities_mapping.each do |key_from_detail, key| | |
keys << key if details[key_from_detail].to_i == 1 | |
end | |
keys | |
end | |
def rentals_from_csv | |
@rentals_from_csv ||= CSV.foreach(rentals_csv_file, csv_options).to_a | |
end | |
def rentals_details_from_csv | |
@rentals_details_from_csv ||= CSV.foreach(rentals_details_csv_file, csv_options).to_a | |
end | |
def addresses_from_csv | |
@addresses_from_csv ||= CSV.foreach(addresses_csv_file, csv_options).to_a | |
end | |
def csv_options | |
{ | |
headers: true, | |
header_converters: :symbol | |
} | |
end | |
def strip(text) | |
ActionView::Base.full_sanitizer.sanitize(text.to_s, tags: []) | |
end | |
class RentalPage | |
attr_reader :rental_page, :url | |
def initialize(mechanize, url) | |
@rental_page = mechanize.get(url) | |
@url = url | |
end | |
def rental_mapping | |
{ | |
url: url, | |
id: id, | |
photos: photos_urls, | |
} | |
end | |
def photos_urls | |
rental_page.search(".structure-photo > a").map { |el| el["href"] } | |
end | |
# http://www.nevache-tourisme.fr/fr/locations-meubles/la-moulini-re-i_47.htm | |
#0. http://www.nevache-tourisme.fr/fr/locations-meubles/la-moulini-re-i_47.htm | |
#1. http://www.nevache-tourisme.fr/fr/locations-meubles/la-moulini-re-i | |
#2. _ | |
#3. 47 | |
#4. .htm | |
def id | |
/(.+)(_)(\d+)(.+)/.match(url)[3] | |
end | |
end | |
class Importer | |
def import!(account, mapping) | |
ActiveRecord::Base.transaction do | |
mapping.each do |rental_mapping| | |
rentals_contact = account.rentals_contacts.where(rental_mapping[:rentals_contact_attributes]).first_or_create | |
if rentals_contact.valid? | |
rental_mapping[:rental_attributes].merge!(contact: rentals_contact) | |
end | |
rental = account.rentals.build(rental_mapping[:rental_attributes]) | |
rental.save! | |
rental_mapping[:photos].each do |photo_url| | |
rental.photos.create!(remote_photo_url: photo_url) | |
end | |
rental_mapping[:rentals_amenities_keys].each do |key| | |
amenity = Amenity.find_by(key: key) | |
RentalsAmenity.create!(amenity: amenity, rental: rental) | |
end | |
end | |
end | |
end | |
end | |
end | |
class Importer | |
def import!(account, mapping) | |
ActiveRecord::Base.transaction do | |
mapping.each do |rental_mapping| | |
rentals_contact = account.rentals_contacts.where(rental_mapping[:rentals_contact_attributes]).first_or_create | |
if rentals_contact.valid? | |
rental_mapping[:rental_attributes].merge!(contact: rentals_contact) | |
end | |
rental = account.rentals.build(rental_mapping[:rental_attributes]) | |
rental.save! | |
rental_mapping[:photos].each do |photo_url| | |
rental.photos.create!(remote_photo_url: photo_url) | |
end | |
rental_mapping[:rentals_amenities_keys].each do |key| | |
amenity = Amenity.find_by(key: key) | |
RentalsAmenity.create!(amenity: amenity, rental: rental) | |
end | |
end | |
end | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment