/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "RealTimeRequestSimulator.h" #include "mozilla/ClearOnShutdown.h" #include "mozilla/Maybe.h" #include "mozilla/Preferences.h" #include "mozilla/RandomNum.h" #include "mozilla/Services.h" #include "mozilla/StaticPrefs_browser.h" #include "mozilla/glean/UrlClassifierMetrics.h" #include "nsIObserverService.h" #include "nsString.h" #include "nsThreadUtils.h" #include "nsUrlClassifierDBService.h" #include "LookupCache.h" #include "prtime.h" namespace mozilla { namespace safebrowsing { StaticRefPtr RealTimeRequestSimulator::sInstance; static constexpr char kContentBlockingCategoryPrefName[] = "browser.contentblocking.category"; StaticAutoPtr sContentBlockingCategory; static constexpr char kRealTimeDebugPrefName[] = "browser.safebrowsing.realTime.debug"; static Maybe sRealTimeDebugEnabled; /* static */ RealTimeRequestSimulator* RealTimeRequestSimulator::GetInstance() { MOZ_ASSERT(NS_IsMainThread()); if (!sInstance) { sInstance = new RealTimeRequestSimulator(); ClearOnShutdown(&sInstance); } return sInstance.get(); } void ContentBlockingCategoryPrefChangeCallback(const char* aPrefName, void*) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(!strcmp(aPrefName, kContentBlockingCategoryPrefName)); MOZ_ASSERT(sContentBlockingCategory); Preferences::GetCString(kContentBlockingCategoryPrefName, *sContentBlockingCategory); } /* static */ const nsCString& RealTimeRequestSimulator::ContentBlockingCategory() { if (!sContentBlockingCategory) { sContentBlockingCategory = new nsCString(); Preferences::RegisterCallbackAndCall( ContentBlockingCategoryPrefChangeCallback, kContentBlockingCategoryPrefName); RunOnShutdown([]() { Preferences::UnregisterCallback(ContentBlockingCategoryPrefChangeCallback, kContentBlockingCategoryPrefName); sContentBlockingCategory = nullptr; }); } return *sContentBlockingCategory; } void RealTimeDebugPrefChangedCallback(const char* aPrefName, void*) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(!strcmp(aPrefName, kRealTimeDebugPrefName)); sRealTimeDebugEnabled = Some(Preferences::GetBool(kRealTimeDebugPrefName, false)); } /* static */ bool RealTimeRequestSimulator::RealTimeDebugEnabled() { if (sRealTimeDebugEnabled.isNothing()) { Preferences::RegisterCallbackAndCall(RealTimeDebugPrefChangedCallback, kRealTimeDebugPrefName); RunOnShutdown([]() { Preferences::UnregisterCallback(RealTimeDebugPrefChangedCallback, kRealTimeDebugPrefName); }); } return sRealTimeDebugEnabled.valueOr(false); } void RealTimeRequestSimulator::ComputeFullHashesFromURL( const nsACString& aURL, nsTArray& aHashes) { nsTArray fragments; LookupCache::GetLookupFragments(aURL, &fragments); aHashes.SetCapacity(fragments.Length()); for (const auto& fragment : fragments) { Completion hash; hash.FromPlaintext(fragment); aHashes.AppendElement(hash); } } // static uint32_t RealTimeRequestSimulator::EstimateRequestSize( uint32_t aHashPrefixCount) { if (aHashPrefixCount == 0) { return 0; } // The request URL format is: // &hashPrefixes=&hashPrefixes=&... // // The gethash URL from browser.safebrowsing.provider.google5.gethashURL is: // https://safebrowsing.googleapis.com/v5/hashes:search?key= // (approximately 70 bytes with the API key) // // For query parameters, each hash prefix contributes: // - "&" separator: 1 byte // - "hashPrefixes=" : 13 bytes // - Base64 encoded 4-byte prefix: 8 bytes (4 bytes -> 6 chars + "==" // padding) // // Total per prefix: 1 + 13 + 8 = 22 bytes constexpr uint32_t kBaseURLSize = 70; constexpr uint32_t kSeparatorSize = 1; // "&" constexpr uint32_t kParamNameSize = 13; // "hashPrefixes=" constexpr uint32_t kBase64PrefixSize = 8; // Base64 encoded 4-byte prefix uint32_t perPrefixSize = kSeparatorSize + kParamNameSize + kBase64PrefixSize; return kBaseURLSize + aHashPrefixCount * perPrefixSize; } // static uint32_t RealTimeRequestSimulator::EstimateResponseSize(uint32_t aNumHits) { // The response is a protobuf-encoded SearchHashesResponse message. // See safebrowsing_v5.proto for the message definition. // // SearchHashesResponse contains: // - repeated FullHash full_hashes = 1; // - Duration cache_duration = 2; // // Protobuf encoding overhead: // - Each field has a tag (1 byte for small field numbers) // - Length-delimited fields (bytes, embedded messages) have a length prefix // // Duration message (cache_duration): // - tag (1 byte) + length (1 byte) + seconds field (~3 bytes for typical // values like 300s) // Estimated: ~5 bytes // // FullHash message (only present on hit): // - full_hash (field 1): tag (1) + length (1) + 32 bytes = 34 bytes // - full_hash_details (field 2): tag (1) + length (1) + FullHashDetail // - FullHashDetail: threat_type enum (~2 bytes) // Estimated: ~4 bytes // - FullHash wrapper: tag (1) + length (1) + content (~38) = ~40 bytes // // Total estimates: // - Miss (no full hashes): ~5 bytes (just cache_duration) // - Hit (N full hashes): ~5 + N * ~40 bytes constexpr uint32_t kCacheDurationSize = 5; constexpr uint32_t kFullHashSize = 40; return kCacheDurationSize + aNumHits * kFullHashSize; } bool RealTimeRequestSimulator::ShouldSimulateHit() { constexpr uint32_t kMaxProbability = 1000000; uint32_t probability = StaticPrefs::browser_safebrowsing_realTime_simulation_hitProbability(); if (probability == 0) { return false; } if (probability >= kMaxProbability) { return true; } Maybe randomVal = RandomUint64(); if (!randomVal) { return false; } return (*randomVal % kMaxProbability) < probability; } void RealTimeRequestSimulator::SimulateRealTimeRequest(const nsACString& aURL, bool aIsPrivate) { MOZ_ASSERT(nsUrlClassifierDBService::BackgroundThread() == NS_GetCurrentThread()); nsTArray fullHashes; ComputeFullHashesFromURL(aURL, fullHashes); int64_t now = PR_Now() / PR_USEC_PER_SEC; bool negativeCacheEnabled = StaticPrefs:: browser_safebrowsing_realTime_simulation_negativeCacheEnabled(); // Filter out cached hashes and check for cache hits. nsTArray hashesToSend; for (const auto& fullHash : fullHashes) { uint32_t prefix = fullHash.ToUint32(); nsCString fullHashString(reinterpret_cast(fullHash.buf), COMPLETE_SIZE); // Check the negative cache first if enabled. if (negativeCacheEnabled) { CachedFullHashResponse* negResponse = mNegativeCache.Get(prefix); if (negResponse) { if (negResponse->negativeCacheExpirySec >= now) { continue; } mNegativeCache.Remove(prefix); } } CachedFullHashResponse* cachedResponse = mSimulatedCache.Get(prefix); if (!cachedResponse) { hashesToSend.AppendElement(fullHash); continue; } // Check if this full hash has a cached hit entry. auto fullHashEntry = cachedResponse->fullHashes.Lookup(fullHashString); if (!fullHashEntry) { hashesToSend.AppendElement(fullHash); continue; } // The full hash cache entry is expired. Remove it and send this hash. if (fullHashEntry.Data() < now) { fullHashEntry.Remove(); if (cachedResponse->fullHashes.IsEmpty()) { mSimulatedCache.Remove(prefix); } hashesToSend.AppendElement(fullHash); continue; } // We find a valid match in the cache, so we don't need to send the // request. NotifyResult(false, 0, 0, aIsPrivate); return; } if (hashesToSend.IsEmpty()) { NotifyResult(false, 0, 0, aIsPrivate); return; } // We will need to simulate a request. Let's estimate the request size. // Include noise entries in the count. uint32_t noiseCount = StaticPrefs::browser_safebrowsing_realTime_simulation_noiseEntryCount(); uint32_t requestBytes = EstimateRequestSize(hashesToSend.Length() + noiseCount); uint32_t ttl = StaticPrefs::browser_safebrowsing_realTime_simulation_cacheTTLSec(); int64_t expiry = now + ttl; uint32_t numHits = 0; uint32_t negativeCacheTTL = StaticPrefs:: browser_safebrowsing_realTime_simulation_negativeCacheTTLSec(); int64_t negativeCacheExpiry = now + negativeCacheTTL; for (const auto& fullHash : hashesToSend) { // If the server doesn't hit the given full hash, we will continue. if (!ShouldSimulateHit()) { if (negativeCacheEnabled) { CachedFullHashResponse* negResponse = mNegativeCache.GetOrInsertNew(fullHash.ToUint32()); negResponse->negativeCacheExpirySec = negativeCacheExpiry; } continue; } numHits++; nsDependentCSubstring fullHashString( reinterpret_cast(fullHash.buf), COMPLETE_SIZE); // There is a hit, so we create a cache entry for it. CachedFullHashResponse* response = mSimulatedCache.GetOrInsertNew(fullHash.ToUint32()); response->fullHashes.InsertOrUpdate(fullHashString, expiry); } // Estimate the response size based on the number of hits. uint32_t responseBytes = EstimateResponseSize(numHits); NotifyResult(true, requestBytes, responseBytes, aIsPrivate); } void RealTimeRequestSimulator::NotifyResult(bool aWouldSendRequest, uint32_t aRequestBytes, uint32_t aResponseBytes, bool aIsPrivate) { NS_DispatchToMainThread(NS_NewRunnableFunction( "RealTimeRequestSimulator::NotifyResult", [aWouldSendRequest, aRequestBytes, aResponseBytes, aIsPrivate]() { if (aWouldSendRequest) { const nsCString& etpCategory = ContentBlockingCategory(); if (!etpCategory.IsEmpty()) { nsAutoCString label; if (etpCategory.EqualsLiteral("standard") || etpCategory.EqualsLiteral("strict") || etpCategory.EqualsLiteral("custom")) { label = etpCategory; } else { label = "other"_ns; } label.Append('_'); label.Append(aIsPrivate ? "private"_ns : "normal"_ns); glean::urlclassifier::realtime_simulation_request_count.Get(label) .Add(1); glean::urlclassifier::realtime_simulation_request_size.Get(label) .Add(aRequestBytes); glean::urlclassifier::realtime_simulation_response_size.Get(label) .Add(aResponseBytes); } } if (RealTimeDebugEnabled()) { nsAutoCString data; data.AppendInt(aWouldSendRequest ? 1 : 0); data.Append(','); data.AppendInt(aRequestBytes); data.Append(','); data.AppendInt(aResponseBytes); nsCOMPtr observerService = mozilla::services::GetObserverService(); if (observerService) { observerService->NotifyObservers( nullptr, "urlclassifier-realtime-simulation-result", NS_ConvertUTF8toUTF16(data).get()); } } })); } void RealTimeRequestSimulator::CleanCache() { MOZ_ASSERT(nsUrlClassifierDBService::BackgroundThread() == NS_GetCurrentThread()); mSimulatedCache.Clear(); mNegativeCache.Clear(); } void RealTimeRequestSimulator::ExpireCache() { MOZ_ASSERT(nsUrlClassifierDBService::BackgroundThread() == NS_GetCurrentThread()); for (auto& entry : mSimulatedCache) { CachedFullHashResponse* response = entry.GetWeak(); response->negativeCacheExpirySec = 0; for (auto iter = response->fullHashes.Iter(); !iter.Done(); iter.Next()) { iter.Data() = 0; } } for (auto& entry : mNegativeCache) { entry.GetWeak()->negativeCacheExpirySec = 0; } } } // namespace safebrowsing } // namespace mozilla