diff --git a/app/models/bike_csv_importer.rb b/app/models/bike_csv_importer.rb index cac2963..f9694b3 100644 --- a/app/models/bike_csv_importer.rb +++ b/app/models/bike_csv_importer.rb @@ -1,6 +1,11 @@ require 'csv' class BikeCsvImporter + + include BikeCsvImporter::Cache + include BikeCsvImporter::Cleaner + include BikeCsvImporter::BikeAttrs + attr_reader :file, :dry_run def initialize(file, dry_run) @@ -11,10 +16,6 @@ class BikeCsvImporter def run result = {imported: {}, skipped: {}} - @bike_purpose_cache = {} - @bike_brand_cache = {} - @bike_model_cache = {} - fetch do |bike_hash| bike = import_bike bike_hash check_method = dry_run ? :valid? : :persisted? @@ -25,12 +26,6 @@ class BikeCsvImporter end end - missing_brands = @bike_brand_cache.select { |_, v| v.nil? }.map(&:first) - result[:missing_brands] = missing_brands if missing_brands.any? - - missing_models = @bike_model_cache.select { |_, v| v.nil? }.map(&:first) - result[:missing_models] = missing_models if missing_models.any? - result end @@ -70,9 +65,6 @@ class BikeCsvImporter @header.zip(row).to_h end - # + Velocipede Number -> Bikes.shop_id - # + Program -> Bikes.bike_purpose_id - # # Gone -> If "Yes", set 'gone' to true, then create a Log entry like the following: # id | loggable_id | loggable_type | logger_id | logger_type | context | start_date | end_date | description | log_action_id | log_action_type | created_at | updated_at # 18 | 1 | Bike | 4 | User | | 2017-02-03 23:27:00 | 2017-02-03 23:27:00 | Gone | 5 | ActsAsLoggable::BikeAction | 2017-02-03 23:27:36.8387 | 2017-02-03 23:27:36.8387 @@ -80,123 +72,11 @@ class BikeCsvImporter # # Date In -> Create a bike log entry with start_date & end_date with same value as "Date In". Set action_id to "AQUIRED" # Date Out -> Should be the start_date & end_date value for "Gone" column mentioned above. - # + Price -> Bikes.value - # + Make -> Bikes.bike_brand_id - # + Model -> Bikes.bike_model_id - # + to Whom -> ignore - # + Zip Code -> ignore # Comment -> Create a bike log entry with action_id "NOTE". The log 'description' should be the value of 'Comment'. - # - # Data at the end of the CSV seems to be missing a lot of fields. If any field is empty, the the value can be "UNDETERMINED" if applicable, or ignored. Any other dates beside "Date In/Out" can be current date. - # - # I realize the log entry stuff is likely complicated and time consuming. At a minimum, the most important columns to import are the following: Velocipede Number, Program, Gone, Make, Model. To avoid creating the log entry for "Gone", we would instead just set 'gone' to true. def import_bike(bike_hash) bike = Bike.new bike_attrs(bike_hash) #bike.save unless dry_run raise 'TODO save' unless dry_run bike end - - def bike_attrs(bike_hash) - %i{ shop_id bike_purpose_id value bike_brand_id bike_model_id model bike_style_id bike_condition_id seat_tube_height bike_wheel_size_id serial_number }.each_with_object({}) do |field, memo| - memo[field] = send :"bike_attr_#{ field }", bike_hash - end - end - - def bike_attr_shop_id(bike_hash) - bike_hash['velocipede number'].to_i - end - - def bike_attr_bike_purpose_id(bike_hash) - map = { - 'SALE' => /shop|as(-|\s+)is|safety\s*check/, - 'BUILDBIKE' => /build|bikes.*world/, - 'STORAGE' => nil, - 'PARTS' => /part|frame/, - 'SCRAP' => /scrap|strip/, - } - - default = 'UNDETERMINED' - test_value = clean_value(bike_hash['program']).try :downcase - value = map.find { |_, regexp| regexp.try :match, test_value }.try :first - - cached_bike_purpose(value || default).id - end - - def bike_attr_gone(bike_hash) - %w{ yes yeah y }.include? clean_value(bike_hash['gone']).try :downcase - end - - def bike_attr_value(bike_hash) - clean_value(bike_hash['price']).try(:gsub, /[$]/, '').try :to_i - end - - def bike_attr_bike_brand_id(bike_hash) - brand = clean_value(bike_hash['make']) - return unless brand - cached_bike_brand(brand).try :id - end - - def bike_attr_bike_model_id(bike_hash) - model = clean_value(bike_hash['model']) - return unless model - cached_bike_model(model).try :id - end - - def bike_attr_model(bike_hash) - model = clean_value bike_hash['model'] - model unless model =~ /unknown/i - end - - def bike_attr_bike_style_id(_) - @bike_style_other_cache ||= BikeStyle.find_by_style('OTHER').id - end - - def bike_attr_bike_condition_id(_) - @bike_condition_undertermined_cache ||= BikeCondition.find_by_condition('UNDETERMINED').id - end - - def bike_attr_seat_tube_height(_) - 0 - end - - def bike_attr_bike_wheel_size_id(_) - @bike_condition_wheel_size_undertermined_cache ||= BikeWheelSize.find_by_description('UNDETERMINED').id - end - - def bike_attr_serial_number(_) - 'UNDETERMINED' - end - - def clean_value(value) - value_or_nil strip_value(value) - end - - def strip_value(value) - value.try(:strip).try(:gsub, /\n|\r/, '') - end - - def value_or_nil(value) - return value unless ['?', 'n/a', 'missing', 'unknown', ''].include? value.try(:downcase) - end - - def cached_bike_purpose(purpose) - @bike_purpose_cache[purpose] ||= BikePurpose.find_by_purpose purpose - end - - def cached_bike_brand(brand) - if @bike_brand_cache.has_key? brand - @bike_brand_cache[brand] - else - @bike_brand_cache[brand] = BikeBrand.where('lower(brand) = ?', brand.downcase).first - end - end - - def cached_bike_model(model) - if @bike_model_cache.has_key? model - @bike_model_cache[model] - else - @bike_model_cache[model] = BikeModel.where('lower(model) = ?', model.downcase).first - end - end end diff --git a/app/models/bike_csv_importer/bike_attrs.rb b/app/models/bike_csv_importer/bike_attrs.rb new file mode 100644 index 0000000..dff8897 --- /dev/null +++ b/app/models/bike_csv_importer/bike_attrs.rb @@ -0,0 +1,78 @@ +class BikeCsvImporter + module BikeAttrs + def bike_attr_fields + %i{ shop_id bike_purpose_id value bike_brand_id bike_model_id model bike_style_id bike_condition_id seat_tube_height bike_wheel_size_id serial_number } + end + + def bike_attrs(bike_hash) + bike_attr_fields.each_with_object({}) do |field, memo| + memo[field] = send :"bike_attr_#{ field }", bike_hash + end + end + + def bike_attr_shop_id(bike_hash) + bike_hash['velocipede number'].to_i + end + + def bike_attr_bike_purpose_id(bike_hash) + map = { + 'SALE' => /shop|as(-|\s+)is|safety\s*check/, + 'BUILDBIKE' => /build|bikes.*world/, + 'STORAGE' => nil, + 'PARTS' => /part|frame/, + 'SCRAP' => /scrap|strip/, + } + + default = 'UNDETERMINED' + test_value = clean_value(bike_hash['program']).try :downcase + value = map.find { |_, regexp| regexp.try :match, test_value }.try :first + + cached_bike_purpose(value || default).id + end + + def bike_attr_gone(bike_hash) + %w{ yes yeah y }.include? clean_value(bike_hash['gone']).try :downcase + end + + def bike_attr_value(bike_hash) + clean_value(bike_hash['price']).try(:gsub, /[$]/, '').try :to_i + end + + def bike_attr_bike_brand_id(bike_hash) + brand = clean_value(bike_hash['make']) + return unless brand + cached_bike_brand(brand).try :id + end + + def bike_attr_bike_model_id(bike_hash) + model = clean_value(bike_hash['model']) + return unless model + cached_bike_model(model).try :id + end + + def bike_attr_model(bike_hash) + model = clean_value bike_hash['model'] + model unless model =~ /unknown/i + end + + def bike_attr_bike_style_id(_) + @bike_style_other_cache ||= BikeStyle.find_by_style('OTHER').id + end + + def bike_attr_bike_condition_id(_) + @bike_condition_undertermined_cache ||= BikeCondition.find_by_condition('UNDETERMINED').id + end + + def bike_attr_seat_tube_height(_) + 0 + end + + def bike_attr_bike_wheel_size_id(_) + @bike_condition_wheel_size_undertermined_cache ||= BikeWheelSize.find_by_description('UNDETERMINED').id + end + + def bike_attr_serial_number(_) + 'UNDETERMINED' + end + end +end diff --git a/app/models/bike_csv_importer/cache.rb b/app/models/bike_csv_importer/cache.rb new file mode 100644 index 0000000..960f37a --- /dev/null +++ b/app/models/bike_csv_importer/cache.rb @@ -0,0 +1,26 @@ +class BikeCsvImporter + module Cache + def cached_bike_purpose(purpose) + @bike_purpose_cache ||= {} + @bike_purpose_cache[purpose] ||= BikePurpose.find_by_purpose purpose + end + + def cached_bike_brand(brand) + @bike_brand_cache ||= {} + if @bike_brand_cache.has_key? brand + @bike_brand_cache[brand] + else + @bike_brand_cache[brand] = BikeBrand.where('lower(brand) = ?', brand.downcase).first + end + end + + def cached_bike_model(model) + @bike_model_cache ||= {} + if @bike_model_cache.has_key? model + @bike_model_cache[model] + else + @bike_model_cache[model] = BikeModel.where('lower(model) = ?', model.downcase).first + end + end + end +end diff --git a/app/models/bike_csv_importer/cleaner.rb b/app/models/bike_csv_importer/cleaner.rb new file mode 100644 index 0000000..1b745eb --- /dev/null +++ b/app/models/bike_csv_importer/cleaner.rb @@ -0,0 +1,15 @@ +class BikeCsvImporter + module Cleaner + def clean_value(value) + value_or_nil strip_value(value) + end + + def strip_value(value) + value.try(:strip).try(:gsub, /\n|\r/, '') + end + + def value_or_nil(value) + return value unless ['?', 'n/a', 'missing', 'unknown', ''].include? value.try(:downcase) + end + end +end