From ec9a97aded91f2ec7c243c962363bb23b6730bc5 Mon Sep 17 00:00:00 2001 From: Ivan Buljan Date: Thu, 5 Dec 2024 17:45:15 +1100 Subject: [PATCH 1/2] Extend 'user' spec factory with 'tijuana_user_with_everything' --- .../spec/factories/tijuana/user.rb | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/spec/test_identity_app/spec/factories/tijuana/user.rb b/spec/test_identity_app/spec/factories/tijuana/user.rb index 5ae702f..78cea49 100644 --- a/spec/test_identity_app/spec/factories/tijuana/user.rb +++ b/spec/test_identity_app/spec/factories/tijuana/user.rb @@ -26,6 +26,29 @@ module IdentityTijuana suburb { Faker::Address.city } postcode { IdentityTijuana::Postcode.new(number: Faker::Address.zip_code, state: Faker::Address.state_abbr) } end + + factory :tijuana_user_with_everything do + home_number { "612#{::Kernel.rand(10_000_000..99_999_999)}" } + mobile_number { "614#{::Kernel.rand(10_000_000..99_999_999)}" } + street_address { Faker::Address.street_address } + suburb { Faker::Address.city } + postcode { IdentityTijuana::Postcode.new(number: Faker::Address.zip_code, state: Faker::Address.state_abbr) } + country_iso { Faker::Address.country_code } + encrypted_password { Faker::Internet.password } + password_salt { Faker::Crypto.md5 } + reset_password_token { Faker::Crypto.md5 } + reset_password_sent_at { 20.days.ago } + remember_created_at { 2.hours.ago } + current_sign_in_at { 1.hours.ago } + last_sign_in_at { 10.days.ago } + last_sign_in_ip { Faker::Internet.ip_v4_address } + random { rand(0.0..0.001) } + notes { Faker::Lorem.paragraph } + quick_donate_trigger_id { Faker::Alphanumeric.alphanumeric(number: 12) } + facebook_id { Faker::Alphanumeric.alphanumeric(number: 12) } + otp_secret_key { Faker::Alphanumeric.alphanumeric(number: 32) } + tracking_token { Faker::Alphanumeric.alphanumeric(number: 8) } + end end end end From 3a98812ce0e5d609d209287429b5bf536a5f6306 Mon Sep 17 00:00:00 2001 From: Ivan Buljan Date: Thu, 5 Dec 2024 17:43:49 +1100 Subject: [PATCH 2/2] Add user anonimisation functionality and tests --- app/lib/identity_tijuana/user_ghosting.rb | 232 ++++++++++++++++++ .../identity_tijuana/anonymisation_log.rb | 6 + lib/identity_tijuana.rb | 12 + spec/app/lib/user_ghosting_spec.rb | 51 ++++ spec/test_identity_app/app/lib/settings.rb | 6 + 5 files changed, 307 insertions(+) create mode 100644 app/lib/identity_tijuana/user_ghosting.rb create mode 100644 app/models/identity_tijuana/anonymisation_log.rb create mode 100644 spec/app/lib/user_ghosting_spec.rb diff --git a/app/lib/identity_tijuana/user_ghosting.rb b/app/lib/identity_tijuana/user_ghosting.rb new file mode 100644 index 0000000..158c761 --- /dev/null +++ b/app/lib/identity_tijuana/user_ghosting.rb @@ -0,0 +1,232 @@ +module IdentityTijuana + class UserGhosting + def initialize(user_ids, reason) + @user_ids = user_ids + @reason = reason + end + + def ghost_users + tj_conn = IdentityTijuana::ReadWrite.connection + log_ids = log_anonymisation_started + begin + tj_conn.transaction do + anon_basic_info(tj_conn) + anon_automation_events(tj_conn) + anon_call_outcomes(tj_conn) + anon_comments(tj_conn) + anon_donations(tj_conn) + anon_event_tracking_logs(tj_conn) + anon_facebook_users(tj_conn) + anon_image_shares(tj_conn) + anon_testimonials(tj_conn) + anon_user_emails(tj_conn) + anon_vision_survey_hashes(tj_conn) + end + rescue ActiveRecord::RecordInvalid => e + log_anonymisation_failed(log_ids, e.message, e) + Rails.logger.error "Anonymisation failed: #{e.message}" + end + + log_anonymisation_finished(log_ids) + end + + private + + def log_anonymisation_started + log_ids = [] + + @user_ids.each do |user_id| + logged = AnonymisationLog.create!( + user_id: user_id, + anonymisation_reason: @reason, + status: 'started', + started_at: DateTime.current.utc, + ) + log_ids << logged.id + end + + log_ids + end + + def log_anonymisation_finished(log_ids) + # rubocop:disable Rails/SkipsModelValidations + AnonymisationLog.where(id: log_ids) + .update_all( + finished_at: DateTime.current.utc, + status: 'completed' + ) + # rubocop:enable Rails/SkipsModelValidations + end + + def log_anonymisation_failed(log_ids, error_msg, error_stack) + # rubocop:disable Rails/SkipsModelValidations + AnonymisationLog.where(id: log_ids) + .update_all( + finished_at: DateTime.current.utc, + status: 'failed', + message: error_msg, + error_stack: error_stack + ) + # rubocop:enable Rails/SkipsModelValidations + end + + # TO DO - make sure that DB schema corresponds + # exist in production and staging, but missing in testing db + # -- new_tags = null, + # -- fragment = null, + # mautic_id = null, + def anon_basic_info(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE users + SET email = concat(id, '@#{Settings.ghoster.email_domain}'), + first_name = null, + last_name = null, + mobile_number = null, + home_number = null, + street_address = null, + country_iso = null, + is_member = 0, + encrypted_password = null, + password_salt = null, + reset_password_token = null, + remember_created_at = null, + sign_in_count = 0, + current_sign_in_at = null, + last_sign_in_at = null, + current_sign_in_ip = null, + last_sign_in_ip = null, + is_admin = 0, + postcode_id = null, + old_tags = '', + is_volunteer = 0, + random = null, + notes = null, + quick_donate_trigger_id = null, + facebook_id = null, + otp_secret_key = null, + tracking_token = null, + do_not_call = 1, + active = 0, + do_not_sms = 1, + updated_at = CURRENT_TIMESTAMP + WHERE id IN (#{@user_ids.join(',')}); + SQL + end + + def anon_automation_events(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE automation_events + SET payload = null + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + + def anon_call_outcomes(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE call_outcomes + SET email = null, + payload = null, + dialed_number = null + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + + # TODO - confirm + def anon_comments(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE comments + SET body = null, + updated_at = CURRENT_TIMESTAMP + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + + def anon_donations(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE donations + SET card_type = null, + card_expiry_month = null, + card_expiry_year = null, + card_last_four_digits = null, + name_on_card = null, + cheque_name = null, + cheque_number = null, + paypal_subscr_id = null, + cancel_reason = null, + identifier = null, + updated_at = CURRENT_TIMESTAMP + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + + # TODO how `user_email_id` relates to PI + # def anon_email_target_tracking_logs(tj_conn) + # tj_conn.execute(<<~SQL.squish) + # UPDATE email_target_tracking_logs + # SET ip = null, + # agent = null, + # cookie = null, + # updated_at = CURRENT_TIMESTAMP + # WHERE user_id IN (#{@user_ids.join(',')}); + # SQL + # end + + def anon_event_tracking_logs(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE event_tracking_logs + SET ip = null, + agent = null, + updated_at = CURRENT_TIMESTAMP + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + + def anon_facebook_users(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE facebook_users + SET facebook_id = null, + updated_at = CURRENT_TIMESTAMP + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + + def anon_image_shares(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE image_shares + SET caption = null, + updated_at = CURRENT_TIMESTAMP + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + + def anon_testimonials(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE testimonials + SET facebook_user_id = null, + updated_at = CURRENT_TIMESTAMP + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + + # TODO + # `user_email` is present in production (missing in testing + staging) + # user_email = null, + def anon_user_emails(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE user_emails + SET body = null, + `from` = null, + updated_at = CURRENT_TIMESTAMP + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + + def anon_vision_survey_hashes(tj_conn) + tj_conn.execute(<<~SQL.squish) + UPDATE vision_survey_hashes + SET `key` = null + WHERE user_id IN (#{@user_ids.join(',')}); + SQL + end + end +end diff --git a/app/models/identity_tijuana/anonymisation_log.rb b/app/models/identity_tijuana/anonymisation_log.rb new file mode 100644 index 0000000..048388f --- /dev/null +++ b/app/models/identity_tijuana/anonymisation_log.rb @@ -0,0 +1,6 @@ +module IdentityTijuana + class AnonymisationLog < ReadWrite + self.table_name = 'anonymisation_logs' + belongs_to :user + end +end diff --git a/lib/identity_tijuana.rb b/lib/identity_tijuana.rb index 28b04c2..9b5dcf5 100644 --- a/lib/identity_tijuana.rb +++ b/lib/identity_tijuana.rb @@ -8,6 +8,18 @@ module IdentityTijuana MEMBER_RECORD_DATA_TYPE = 'object'.freeze MUTEX_EXPIRY_DURATION = 10.minutes + def self.ghost_members(member_ids:, reason:, admin_member_id:) + return if member_ids.empty? + + # check that we have the matching user ids for TJ system + user_ids = MemberExternalId.where(system: 'tijuana', member_id: member_ids) + .pluck(:external_id) + .map(&:to_i) + + anon_reason = "#{reason} - via Id - admin:#{admin_member_id}" + UserGhosting.new(user_ids, anon_reason).ghost_users if user_ids.any? + end + def self.push(_sync_id, member_ids, _external_system_params) members = Member.where(id: member_ids).with_email.order(:id) yield members, nil diff --git a/spec/app/lib/user_ghosting_spec.rb b/spec/app/lib/user_ghosting_spec.rb new file mode 100644 index 0000000..96aef96 --- /dev/null +++ b/spec/app/lib/user_ghosting_spec.rb @@ -0,0 +1,51 @@ +require 'rails_helper' + +describe IdentityTijuana::UserGhosting do + before(:each) do + allow(Settings).to( + receive_message_chain("tijuana.database_url") { ENV['TIJUANA_DATABASE_URL'] } + ) + allow(Settings).to( + receive_message_chain("ghoster.email_domain") { 'anoned.non' } + ) + end + + context '#ghosting' do + context 'user with all attributes' do + it 'should ghost all attributes' do + u = FactoryBot.create(:tijuana_user_with_everything) + + anon_domain = Settings.ghoster.email_domain + described_class.new([u.id], 'test-reason').ghost_users + + u.reload + + expect(u.email).to eq("#{u.id}@#{anon_domain}") + + nils = [:first_name, :last_name, :mobile_number, + :home_number, :street_address, :country_iso, + :encrypted_password, :password_salt, :reset_password_token, + :remember_created_at, :current_sign_in_at, :last_sign_in_at, + :current_sign_in_ip, :last_sign_in_ip, :postcode_id, + :random, :notes, :quick_donate_trigger_id, :facebook_id, + :otp_secret_key, :tracking_token] + + nils.each do |prop| + expect(u.send(prop)).to be(nil) + end + + falsey = [:is_member, :is_admin, :active, :is_volunteer] + + falsey.each do |prop| + expect(u.send(prop)).to eq(false) + end + + truthy = [:do_not_call, :do_not_sms] + + truthy.each do |prop| + expect(u.send(prop)).to eq(true) + end + end + end + end +end diff --git a/spec/test_identity_app/app/lib/settings.rb b/spec/test_identity_app/app/lib/settings.rb index 6ace66e..fa22d95 100644 --- a/spec/test_identity_app/app/lib/settings.rb +++ b/spec/test_identity_app/app/lib/settings.rb @@ -38,6 +38,12 @@ def self.databases } end + def self.ghoster + return { + "email_domain" => "example.com" + } + end + def self.redis_url return ENV['REDIS_URL'] end