diff --git a/Steepfile b/Steepfile index c8fc46dc..d81fac41 100644 --- a/Steepfile +++ b/Steepfile @@ -18,12 +18,10 @@ target :lib do # Use the stubs located at sig/custom/.rbs instead. library 'date' library 'fileutils' - library 'ipaddr' library 'logger' library 'json' library 'openssl' library 'pathname' - library 'resolv' library 'tempfile' library 'time' library 'uri' diff --git a/lib/mindee/http/cancellation_token.rb b/lib/mindee/http/cancellation_token.rb index 563d4d93..6dfcae70 100644 --- a/lib/mindee/http/cancellation_token.rb +++ b/lib/mindee/http/cancellation_token.rb @@ -7,17 +7,17 @@ class CancellationToken private attr_reader :is_canceled def initialize - @is_canceled = false + @is_cancelled = false end # Cancel the token. def cancel - @is_canceled = true + @is_cancelled = true end # Check if the token is canceled. def canceled? - @is_canceled + @is_cancelled end end end diff --git a/lib/mindee/input/sources/url_input_source.rb b/lib/mindee/input/sources/url_input_source.rb index dacc3d15..3947c723 100644 --- a/lib/mindee/input/sources/url_input_source.rb +++ b/lib/mindee/input/sources/url_input_source.rb @@ -3,8 +3,6 @@ require 'net/http' require 'uri' require 'fileutils' -require 'ipaddr' -require 'resolv' require_relative '../../logging' module Mindee @@ -12,27 +10,14 @@ module Input module Source # Load a remote document from a file url. class URLInputSource - # IP ranges that must not be the target of a source URL (SSRF protection). - DISALLOWED_RANGES = [ - IPAddr.new('127.0.0.0/8'), # IPv4 loopback - IPAddr.new('::1/128'), # IPv6 loopback - IPAddr.new('169.254.0.0/16'), # IPv4 link-local - IPAddr.new('fe80::/10'), # IPv6 link-local - IPAddr.new('10.0.0.0/8'), # RFC 1918 - IPAddr.new('172.16.0.0/12'), # RFC 1918 - IPAddr.new('192.168.0.0/16'), # RFC 1918 - IPAddr.new('0.0.0.0/8'), # "this" network - IPAddr.new('224.0.0.0/4'), # IPv4 multicast - IPAddr.new('ff00::/8'), # IPv6 multicast - IPAddr.new('fc00::/7'), # IPv6 unique-local - IPAddr.new('100.64.0.0/10'), # Carrier-grade NAT (RFC 6598) - ].freeze # @return [String] attr_reader :url def initialize(url) - validate_url!(url) + raise Error::MindeeInputError, 'URL must be HTTPS' unless url.start_with? 'https://' + logger.debug("URL input: #{url}") + @url = url end @@ -100,62 +85,6 @@ def fetch_file_content(username: nil, password: nil, token: nil, max_redirects: private - # Validates the URL against SSRF risks before it is stored or fetched. - # Checks scheme, embedded credentials, loopback hostnames, and resolved addresses. - # @param url [String] - # @raise [Error::MindeeInputError] - # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity - def validate_url!(url) - uri = URI.parse(url) - - raise Error::MindeeInputError, 'Only HTTPS source URLs are allowed' unless uri.scheme&.downcase == 'https' - - userinfo = uri.userinfo - raise Error::MindeeInputError, 'Source URLs must not embed user credentials' if userinfo && !userinfo.empty? - - host = uri.hostname - raise Error::MindeeInputError, 'Source URL is missing a host' if host.nil? || host.empty? - - lower_host = host.downcase - if lower_host == 'localhost' || lower_host.end_with?('.localhost') || - lower_host == 'ip6-localhost' || lower_host == 'ip6-loopback' - raise Error::MindeeInputError, "Loopback hostnames are not allowed: #{host}" - end - - resolved_addresses(host).each do |addr| - if DISALLOWED_RANGES.any? { |range| range.include?(addr) } - raise Error::MindeeInputError, - "Source URL host resolves to a disallowed address: #{addr}" - end - end - rescue URI::InvalidURIError - raise Error::MindeeInputError, "Invalid URL: #{url}" - end - # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity - - # Resolves a host to an array of IPAddr objects. - # Handles literal IP addresses directly without a DNS round-trip. - # @param host [String] - # @return [Array] - # @raise [Error::MindeeInputError] if the host cannot be resolved. - def resolved_addresses(host) - return [IPAddr.new(host)] if literal_ip?(host) - - addrs = Resolv.getaddresses(host) - raise Error::MindeeInputError, "Unable to resolve source URL host: #{host}" if addrs.empty? - - addrs.map { |a| IPAddr.new(a) } - end - - # Returns true if the string is a valid literal IPv4 or IPv6 address. - # @param host [String] - def literal_ip?(host) - IPAddr.new(host) - true - rescue IPAddr::InvalidAddressError, IPAddr::AddressFamilyError - false - end - def extract_filename_from_url(uri) filename = File.basename(uri.path.to_s) filename.empty? ? '' : filename @@ -176,7 +105,6 @@ def make_request(uri, request, max_redirects) location = response['location'] raise Error::MindeeInputError, 'No location in redirection header.' if location.nil? - validate_url!(location) new_uri = URI.parse(location) request = Net::HTTP::Get.new(new_uri) make_request(new_uri, request, max_redirects - 1) diff --git a/lib/mindee/v2/parsing/field/field_confidence.rb b/lib/mindee/v2/parsing/field/field_confidence.rb index c19bf313..857c9b21 100644 --- a/lib/mindee/v2/parsing/field/field_confidence.rb +++ b/lib/mindee/v2/parsing/field/field_confidence.rb @@ -133,7 +133,7 @@ def <(other) # Aliases for the comparison operators. alias eql? == - alias equal_value? == + alias equal? == alias gteql? >= alias greater_than_or_equal? >= alias lteql? <= diff --git a/sig/mindee/http/cancellation_token.rbs b/sig/mindee/http/cancellation_token.rbs index e0efe700..0ad18b4a 100644 --- a/sig/mindee/http/cancellation_token.rbs +++ b/sig/mindee/http/cancellation_token.rbs @@ -3,7 +3,7 @@ module Mindee module HTTP class CancellationToken - @is_canceled: bool + @is_cancelled: bool attr_reader is_canceled: bool diff --git a/sig/mindee/input/sources/url_input_source.rbs b/sig/mindee/input/sources/url_input_source.rbs index 174e69ce..77d2e036 100644 --- a/sig/mindee/input/sources/url_input_source.rbs +++ b/sig/mindee/input/sources/url_input_source.rbs @@ -3,8 +3,6 @@ module Mindee module Input module Source class URLInputSource - DISALLOWED_RANGES: Array[IPAddr] - def url: -> String def logger: () -> Logger def initialize: (String) -> void @@ -16,9 +14,6 @@ module Mindee def make_request: (URI::Generic, Net::HTTP::Get, Integer) -> Net::HTTPRedirection def get_file_extension: (String) -> String? def generate_file_name: (?extension: String?) -> String - def validate_url!: (String) -> void - def resolved_addresses: (String) -> Array[IPAddr] - def literal_ip?: (String) -> bool end end end diff --git a/sig/mindee/v2/parsing/field/field_confidence.rbs b/sig/mindee/v2/parsing/field/field_confidence.rbs index 6eb288dc..f1715d98 100644 --- a/sig/mindee/v2/parsing/field/field_confidence.rbs +++ b/sig/mindee/v2/parsing/field/field_confidence.rbs @@ -28,7 +28,7 @@ module Mindee def lt?: (String | Integer | FieldConfidence) -> bool def less_than?: (String | Integer | FieldConfidence) -> bool def less_than_or_equal?: (String | Integer | FieldConfidence) -> bool - def equal_value?: (String | Integer | FieldConfidence) -> bool + def equal?: (String | Integer | FieldConfidence) -> bool def inspect: -> String def val_to_i: (String) -> Integer diff --git a/spec/input/sources/url_input_source_spec.rb b/spec/input/sources/url_input_source_spec.rb index 920d1d29..87407af0 100644 --- a/spec/input/sources/url_input_source_spec.rb +++ b/spec/input/sources/url_input_source_spec.rb @@ -9,12 +9,6 @@ let(:invalid_url) { 'http://invalidurl/some/file.jpg' } let(:output_dir) { "#{ROOT_DATA_DIR}/output/" } - # Stub DNS for hosts used in the existing tests so they pass validation. - before do - allow(Resolv).to receive(:getaddresses).with('validurl').and_return(['1.2.3.4']) - allow(Resolv).to receive(:getaddresses).with('platform.mindee.com').and_return(['1.2.3.4']) - end - describe '#initialize' do context 'with valid URL' do it 'creates a new instance' do @@ -71,143 +65,6 @@ end end - describe '#validate_url!' do - let(:safe_url) { 'https://safe.example.com/file.pdf' } - - before do - allow(Resolv).to receive(:getaddresses).with('safe.example.com').and_return(['1.2.3.4']) - end - - context 'scheme' do - it 'accepts https' do - expect { described_class.new(safe_url) }.not_to raise_error - end - - it 'rejects http' do - expect { described_class.new('http://safe.example.com/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{Only HTTPS}) - end - - it 'rejects ftp' do - expect { described_class.new('ftp://safe.example.com/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{Only HTTPS}) - end - end - - context 'embedded credentials' do - it 'rejects URLs with user:password' do - allow(Resolv).to receive(:getaddresses).with('safe.example.com').and_return(['1.2.3.4']) - expect { described_class.new('https://user:pass@safe.example.com/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{credentials}) - end - - it 'rejects URLs with username only' do - allow(Resolv).to receive(:getaddresses).with('safe.example.com').and_return(['1.2.3.4']) - expect { described_class.new('https://user@safe.example.com/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{credentials}) - end - end - - context 'loopback hostnames' do - it 'rejects localhost' do - expect { described_class.new('https://localhost/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{Loopback}) - end - - it 'rejects subdomains of localhost' do - expect { described_class.new('https://evil.localhost/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{Loopback}) - end - - it 'rejects ip6-localhost' do - expect { described_class.new('https://ip6-localhost/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{Loopback}) - end - - it 'rejects ip6-loopback' do - expect { described_class.new('https://ip6-loopback/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{Loopback}) - end - end - - context 'literal IP addresses' do - it 'rejects IPv4 loopback (127.0.0.1)' do - expect { described_class.new('https://127.0.0.1/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects any 127.x.x.x address' do - expect { described_class.new('https://127.0.0.2/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects IPv6 loopback (::1)' do - expect { described_class.new('https://[::1]/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects RFC 1918 — 10.x.x.x' do - expect { described_class.new('https://10.0.0.1/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects RFC 1918 — 172.16.x.x' do - expect { described_class.new('https://172.16.0.1/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects RFC 1918 — 192.168.x.x' do - expect { described_class.new('https://192.168.1.1/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects link-local — 169.254.x.x' do - expect { described_class.new('https://169.254.0.1/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects carrier-grade NAT — 100.64.x.x' do - expect { described_class.new('https://100.64.0.1/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects any-local — 0.0.0.0' do - expect { described_class.new('https://0.0.0.0/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects IPv6 unique-local (fc00::1)' do - expect { described_class.new('https://[fc00::1]/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects IPv6 link-local (fe80::1)' do - expect { described_class.new('https://[fe80::1]/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - end - - context 'hostname resolving to a disallowed address' do - it 'rejects a hostname that resolves to a private IP' do - allow(Resolv).to receive(:getaddresses).with('internal.corp').and_return(['192.168.1.50']) - expect { described_class.new('https://internal.corp/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects a hostname that resolves to a loopback IP' do - allow(Resolv).to receive(:getaddresses).with('fake-local.example.com').and_return(['127.0.0.1']) - expect { described_class.new('https://fake-local.example.com/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{disallowed address}) - end - - it 'rejects an unresolvable hostname' do - allow(Resolv).to receive(:getaddresses).with('unresolvable.invalid').and_return([]) - expect { described_class.new('https://unresolvable.invalid/file.pdf') } - .to raise_error(Mindee::Error::MindeeInputError, %r{Unable to resolve}) - end - end - end - describe '#write_to_file' do let(:url_input_source) { described_class.new(valid_url) } let(:url_input_source_no_filename) { described_class.new(valid_url_no_filename) } diff --git a/spec/v2/parsing/field_confidence_spec.rb b/spec/v2/parsing/field_confidence_spec.rb index 4c3421ca..c4004587 100644 --- a/spec/v2/parsing/field_confidence_spec.rb +++ b/spec/v2/parsing/field_confidence_spec.rb @@ -106,25 +106,25 @@ end end - describe '#equal_value? / #eql?' do + describe '#equal? / #eql?' do it 'returns true for equal confidence values' do - expect(high.equal_value?(described_class.new('High'))).to be true + expect(high.equal?(described_class.new('High'))).to be true expect(high.eql?(described_class.new('High'))).to be true end it 'returns false for different confidence values' do - expect(high.equal_value?(medium)).to be false + expect(high.equal?(medium)).to be false expect(high.eql?(low)).to be false end it 'compares against a String' do - expect(high.equal_value?('High')).to be true - expect(high.equal_value?('Medium')).to be false + expect(high.equal?('High')).to be true + expect(high.equal?('Medium')).to be false end it 'compares against an Integer' do - expect(high.equal_value?(3)).to be true - expect(high.equal_value?(2)).to be false + expect(high.equal?(3)).to be true + expect(high.equal?(2)).to be false end end end