diff --git a/.rubocop.yml b/.rubocop.yml index 987882b2..a949a4e4 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -30,6 +30,7 @@ Metrics/BlockLength: - 'test/**/*' - 'lib/classifier/extensions/vector.rb' - 'lib/classifier/cli.rb' + - 'lib/classifier/keywords/cli.rb' # Allow longer methods in complex algorithms (SVD, etc.) and CLI Metrics/MethodLength: @@ -39,6 +40,7 @@ Metrics/MethodLength: - 'lib/classifier/extensions/vector.rb' - 'lib/classifier/lsi/content_node.rb' - 'lib/classifier/cli.rb' + - 'lib/classifier/keywords/cli.rb' # Allow higher complexity for mathematical algorithms and CLI Metrics/AbcSize: @@ -49,6 +51,7 @@ Metrics/AbcSize: - 'lib/classifier/lsi.rb' - 'lib/classifier/lsi/content_node.rb' - 'lib/classifier/cli.rb' + - 'lib/classifier/keywords/cli.rb' Metrics/CyclomaticComplexity: Max: 10 @@ -63,6 +66,7 @@ Metrics/PerceivedComplexity: - 'lib/classifier/extensions/vector.rb' - 'lib/classifier/lsi/content_node.rb' - 'lib/classifier/cli.rb' + - 'lib/classifier/keywords/cli.rb' # Class length limits - algorithms, tests and CLI can be longer Metrics/ClassLength: diff --git a/README.md b/README.md index b783fdfa..0089a091 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,49 @@ classifier "Great product, highly recommend" # => positive ``` +Extract keywords and analyze term importance using TF-IDF instantly: + +```bash +# Extract from a raw string +keywords "Ruby is a programming language" +# => ruby:0.52 programming:0.41 language:0.38 + +# Extract from a file +keywords extract article.txt +# => machine:0.61 learning:0.58 neural:0.45 network:0.42 + +# Pipeline with stdin and web data +curl -s https://example.com/article | keywords extract + +# Get top 5 terms only +keywords -n 5 "long document with many terms..." + +# Use a custom model file +keywords -m custom_model.json "Ruby is a programming language" +``` + +Build your own vocabulary (fit data): +```bash +# Fit from multiple files +keywords fit corpus/*.txt + +# Fit from stdin (each line is treated as a separate document) +cat documents.txt | keywords fit + +# Tune vocabulary filters during fitting +keywords fit --min-df 2 --max-df 0.85 --ngram 1,2 corpus/*.txt +``` + +Inspect your model: +```bash +# Check model statistics and parameters +keywords info +# => Documents: 1,234 +# => Vocabulary: 5,678 +# => Min DF: 1 +# => Max DF: 1.0 +``` + [CLI Guide →](https://rubyclassifier.com/docs/guides/cli/basics) ### Claude Code Plugin diff --git a/classifier.gemspec b/classifier.gemspec index 9ebfbbe0..0a0a4321 100644 --- a/classifier.gemspec +++ b/classifier.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.required_ruby_version = '>= 3.1' s.files = Dir['{lib,sig,exe}/**/*.{rb,rbs}', 'ext/**/*.{c,h,rb}', 'exe/*', 'bin/*', 'LICENSE', '*.md', 'test/*'] s.bindir = 'exe' - s.executables = ['classifier'] + s.executables = %w[classifier keywords] s.extensions = ['ext/classifier/extconf.rb'] s.license = 'LGPL' diff --git a/exe/classifier b/exe/classifier index 49ecd105..e5ea4450 100755 --- a/exe/classifier +++ b/exe/classifier @@ -1,7 +1,6 @@ #!/usr/bin/env ruby # frozen_string_literal: true -# Force UTF-8 encoding for proper handling of model data and user input Encoding.default_external = Encoding::UTF_8 Encoding.default_internal = Encoding::UTF_8 diff --git a/exe/keywords b/exe/keywords new file mode 100755 index 00000000..eafb9e68 --- /dev/null +++ b/exe/keywords @@ -0,0 +1,15 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Force UTF-8 encoding for proper handling of model data and user input +Encoding.default_external = Encoding::UTF_8 +Encoding.default_internal = Encoding::UTF_8 + +require 'classifier/keywords/cli' + +result = Classifier::Keywords::CLI.new(ARGV).run + +warn result[:error] unless result[:error].empty? +puts result[:output] unless result[:output].empty? + +exit result[:exit_code] diff --git a/lib/classifier/extensions/word_hash.rb b/lib/classifier/extensions/word_hash.rb index c51a8d8e..6dac1ab6 100644 --- a/lib/classifier/extensions/word_hash.rb +++ b/lib/classifier/extensions/word_hash.rb @@ -33,6 +33,12 @@ def clean_word_hash(min_word_length = 3) word_hash_for_words(gsub(/[^\w\s]/, '').split, min_word_length) end + # Builds a mapping between stemmed roots and their most frequent original words. + # @rbs (?Integer) -> Hash[Symbol, String] + def stem_to_word_hash(min_word_length = 3) + mapping_stem_to_word_for_words(gsub(/[^\w\s]/, '').split, min_word_length) + end + private # @rbs (Array[String], Integer) -> Hash[Symbol, Integer] @@ -54,6 +60,20 @@ def word_hash_for_symbols(words) d end + # @rbs (Array[String], Integer) -> Hash[Symbol, Integer] + def mapping_stem_to_word_for_words(words, min_word_length) + h = {} + words.map { _1.tap(&:downcase!) }.tally.each do |word, count| + next unless !CORPUS_SKIP_WORDS.include?(word) && word.length >= min_word_length + + stem = word.stem.intern + h[stem] ||= [word, count] + h[stem] = [word, count] if h.dig(stem, 1) < count + end + h.each_key { |k| h[k] = h[k].first } + h + end + CORPUS_SKIP_WORDS = ::Set.new(%w[ a again diff --git a/lib/classifier/keywords/cli.rb b/lib/classifier/keywords/cli.rb new file mode 100644 index 00000000..7f86f68f --- /dev/null +++ b/lib/classifier/keywords/cli.rb @@ -0,0 +1,272 @@ +# rbs_inline: enabled + +require 'optparse' +require 'fileutils' +require 'stringio' +require 'classifier' + +module Classifier + module Keywords + class CLI + class UsageError < StandardError; end + + # @rbs @args: Array[String] + # @rbs @stdin: String? + # @rbs @options: Hash[Symbol, untyped] + # @rbs @output: Array[String] + # @rbs @error: Array[String] + # @rbs @exit_code: Integer + # @rbs @parser: OptionParser + + def initialize(args, stdin: nil) + @args = args.dup + @stdin = stdin + @options = { + model: File.expand_path('./keywords.json'), + top: nil, + quiet: false, + min_df: 1, + max_df: 1.0, + ngram_range: [1, 1] + } + @output = [] + @error = [] + @exit_code = 0 + end + + def run + parse_options + execute_command + { output: @output.join("\n"), error: @error.join("\n"), exit_code: @exit_code } + rescue OptionParser::InvalidOption, OptionParser::MissingArgument, + OptionParser::InvalidArgument, UsageError => e + @error << "Error: #{e.message}" + @exit_code = 2 + { output: @output.join("\n"), error: @error.join("\n"), exit_code: @exit_code } + rescue StandardError => e + @error << "Error: #{e.message}" + @exit_code = 1 + { output: @output.join("\n"), error: @error.join("\n"), exit_code: @exit_code } + end + + private + + def parse_options + @parser = OptionParser.new do |opts| + opts.banner = 'Usage: keywords [text] [options] [command] [arguments]' + opts.separator '' + opts.separator 'Commands:' + opts.separator ' fit Fit the model from files or stdin (each line is treated as a separate document)' + opts.separator ' extract Extract keywords from a file' + opts.separator ' info Show model information' + opts.separator ' Get weighted terms from text' + opts.separator '' + opts.separator 'Options:' + + opts.on('-m', '--model FILE', 'Model file (default: ./keywords.json)') do |file| + @options[:model] = File.expand_path(file) + end + + opts.on('-n', '--top N', Integer, 'Show top N terms only') do |n| + raise OptionParser::InvalidArgument, 'must be positive' unless n.positive? + + @options[:top] = n + end + + opts.on('--min-df N', Integer, 'Minimum document frequency (default: 1)') do |n| + @options[:min_df] = n + end + + opts.on('--max-df N', Float, 'Maximum document frequency ratio (default: 1.0)') do |n| + @options[:max_df] = n + end + + opts.on('--ngram MIN,MAX', Array, 'N-gram range (default: 1,1)') do |range| + raise OptionParser::InvalidArgument, 'requires exactly two values' if range.count != 2 + + raise OptionParser::InvalidArgument, 'must be integers' unless range.all? { |n| n =~ /\A\d+\z/ } + + min, max = range.map(&:to_i) + + raise OptionParser::InvalidArgument, 'bounds must be >= 1 and min <= max' if min < 1 || max < 1 || min > max + + @options[:ngram_range] = [min, max] + end + + opts.on('-q', 'Quiet mode') do + @options[:quiet] = true + end + + opts.on('-v', '--version', 'Show version') do + @output << Classifier::VERSION + @exit_code = 0 + throw :done + end + + opts.on('-h', '--help', 'Show help') do + @output << opts.to_s + @exit_code = 0 + throw :done + end + end + + catch(:done) do + @parser.parse!(@args) + end + end + + def execute_command + return if @exit_code != 0 || @output.any? + + command = @args.first + + case command + when 'fit' + command_fit + when 'extract' + command_extract + when 'info' + command_info + else + command_keywords + end + end + + def command_fit + # @type var streams: Array[IO | StringIO] + streams = [] + + @args.shift + + streams = + if @args.empty? + [@stdin ? StringIO.new(@stdin.to_s) : $stdin] + else + files = @args.map { |arg| Dir.glob(arg).map { |f| File.expand_path(f) } }.flatten.uniq + files.map { |f| File.open(f) } + end + + tfidf = TFIDF.new( + min_df: @options[:min_df], + max_df: @options[:max_df], + ngram_range: @options[:ngram_range] + ) + tfidf.fit_from_stream(Streaming::MultiIO.new(streams)) + + raise UsageError, 'No documents found to save the model' if tfidf.num_documents.zero? + + tfidf.save_to_file(@options[:model]) + @output << "Saved to #{@options[:model].inspect}" unless @options[:quiet] + ensure + streams.each(&:close) unless @args.empty? + end + + def command_extract + @args.shift + + document = + if @args.empty? + @stdin ? @stdin.to_s : $stdin.read + else + file = File.expand_path(@args.first) + File.exist?(file) ? File.read(file) : @args.first + end + + transform(document) + end + + def command_info + @args.shift + + tfidf = TFIDF.load_from_file(@options[:model]) + documents = number_with_delimiter(tfidf.num_documents) + vocabulary = number_with_delimiter(tfidf.vocabulary.count) + min_df = tfidf.min_df + max_df = tfidf.max_df + @output << format( + "Documents: %s\nVocabulary: %s\n" \ + "Min DF: %d\nMax DF: %.1f", + documents: documents, vocabulary: vocabulary, min_df: min_df, max_df: max_df + ) + end + + def command_keywords + if @args.empty? && @stdin.nil? && $stdin.tty? + show_getting_started + return + end + + document = + if @args.empty? + @stdin ? @stdin.to_s : $stdin.read + else + @args.join(' ') + end + + transform(document) + end + + def show_getting_started + @output << 'Keywords - Keyword extraction and term analysis using TF-IDF' + @output << '' + @output << 'Get started by building a vocabulary (fitting data):' + @output << '' + @output << ' # Fit from files' + @output << ' keywords fit corpus/*.txt' + @output << '' + @output << ' # Fit from stdin' + @output << ' cat documents.txt | keywords fit' + @output << '' + @output << ' # Keep in mind that each line is treated as a separate document.' + @output << '' + @output << 'Then extract weighted terms and analyze text:' + @output << '' + @output << ' # Extract from string' + @output << " keywords 'Ruby is a programming language'" + @output << ' # ruby:0.52 programming:0.41 language:0.38' + @output << '' + @output << ' # Extract from file (convenience alias)' + @output << ' keywords extract article.txt' + @output << '' + @output << ' # Pipeline with stdin and web data' + @output << ' curl -s https://example.com/article | keywords extract' + @output << '' + @output << 'Check model statistics:' + @output << '' + @output << ' keywords info' + @output << ' # Documents: 1,234' + @output << ' # Vocabulary: 5,678' + @output << ' # Min DF: 1' + @output << ' # Max DF: 1.0' + @output << '' + @output << 'General Options:' + @output << ' -m, --model FILE Model file (default: ./keywords.json)' + @output << ' -n, --top N Show top N terms only (e.g. keywords -n 5 "text...")' + @output << '' + @output << 'Fit-specific Options:' + @output << ' --min-df N Minimum document frequency (default: 1)' + @output << ' --max-df N Maximum document frequency ratio (default: 1.0)' + @output << ' --ngram MIN,MAX N-gram range (default: 1,1)' + @output << '' + @output << 'Run "keywords --help" for full usage.' + end + + def transform(document) + unless File.exist?(@options[:model]) + raise UsageError, "No model found; run 'keywords fit' first or " \ + "pass correct model using the '-m' option." + end + + stem_map = document.stem_to_word_hash + tfidf = TFIDF.load_from_file(@options[:model]) + vector = tfidf.transform(document).sort_by { |_, v| v }.reverse + vector = vector.first(@options[:top]) if @options[:top] + @output << vector.map { |k, v| "#{stem_map[k]}:#{v.round(2)}" }.join(' ') + end + + def number_with_delimiter(number, delimiter: ',') + number.to_s.reverse.scan(/\d{1,3}/).join(delimiter).reverse + end + end + end +end diff --git a/lib/classifier/streaming.rb b/lib/classifier/streaming.rb index 72382670..8330d7cf 100644 --- a/lib/classifier/streaming.rb +++ b/lib/classifier/streaming.rb @@ -2,6 +2,7 @@ require_relative 'streaming/progress' require_relative 'streaming/line_reader' +require_relative 'streaming/multi_io' module Classifier # Streaming module provides memory-efficient training capabilities for classifiers. diff --git a/lib/classifier/streaming/line_reader.rb b/lib/classifier/streaming/line_reader.rb index e0b92f31..17ad93b0 100644 --- a/lib/classifier/streaming/line_reader.rb +++ b/lib/classifier/streaming/line_reader.rb @@ -15,14 +15,14 @@ module Streaming class LineReader include Enumerable #[String] - # @rbs @io: IO + # @rbs @io: IO | Classifier::Streaming::MultiIO # @rbs @batch_size: Integer attr_reader :batch_size # Creates a new LineReader. # - # @rbs (IO, ?batch_size: Integer) -> void + # @rbs (IO | Classifier::Streaming::MultiIO, ?batch_size: Integer) -> void def initialize(io, batch_size: 100) @io = io @batch_size = batch_size @@ -68,27 +68,29 @@ def each_batch def estimate_line_count(sample_size: 100) return nil unless @io.respond_to?(:size) && @io.respond_to?(:rewind) + io = @io #: ::IO + begin - original_pos = @io.pos - @io.rewind + original_pos = io.pos + io.rewind sample_bytes = 0 sample_lines = 0 sample_size.times do - line = @io.gets + line = io.gets break unless line sample_bytes += line.bytesize sample_lines += 1 end - @io.seek(original_pos) + io.seek(original_pos) return nil if sample_lines.zero? avg_line_size = sample_bytes.to_f / sample_lines - io_size = @io.__send__(:size) #: Integer + io_size = io.__send__(:size) #: Integer (io_size / avg_line_size).round rescue IOError, Errno::ESPIPE nil diff --git a/lib/classifier/streaming/multi_io.rb b/lib/classifier/streaming/multi_io.rb new file mode 100644 index 00000000..bc71681a --- /dev/null +++ b/lib/classifier/streaming/multi_io.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true +# rbs_inline: enabled + +module Classifier + module Streaming + # A utility class that wraps multiple IO-like streams and treats them as a single, + # sequential stream. + # + # `MultiIO` allows you to iterate over lines across multiple input sources + # (such as files, standard input, or string buffers) seamlessly in the order + # they are provided. + # + # @example Reading from multiple files sequentially + # log1 = File.open("syslog.log") + # log2 = File.open("auth.log") + # + # multi = MultiIO.new([log1, log2]) + # multi.each_line do |line| + # puts line if line.include?("ERROR") + # end + # + class MultiIO + # @rbs @streams: Array[IO] + + def initialize(streams) + @streams = streams.dup + end + + # @rbs () { (String) -> void } -> void + # @rbs () -> Enumerator[String, untyped] + def each_line + # rubocop:disable Style/ExplicitBlockArgument + return enum_for(:each_line) unless block_given? + + @streams.each do |stream| + stream.each_line { |line| yield line } + end + # rubocop:enable Style/ExplicitBlockArgument + end + end + end +end diff --git a/lib/classifier/tfidf.rb b/lib/classifier/tfidf.rb index 20a65e11..09b647c6 100644 --- a/lib/classifier/tfidf.rb +++ b/lib/classifier/tfidf.rb @@ -30,7 +30,7 @@ class TFIDF # @rbs @storage: Storage::Base? # @rbs @min_word_length: Integer - attr_reader :vocabulary, :idf, :num_documents + attr_reader :vocabulary, :idf, :num_documents, :min_df, :max_df attr_accessor :storage # Creates a new TF-IDF vectorizer. @@ -285,7 +285,8 @@ def self.load_checkpoint(storage:, checkpoint_id:) # puts "#{progress.completed} documents loaded" # end # - # @rbs (IO, ?batch_size: Integer) { (Streaming::Progress) -> void } -> self + # @rbs (IO | Classifier::Streaming::MultiIO, ?batch_size: Integer) -> self + # @rbs (IO | Classifier::Streaming::MultiIO, ?batch_size: Integer) { (Streaming::Progress) -> void } -> self def fit_from_stream(io, batch_size: Streaming::DEFAULT_BATCH_SIZE) reader = Streaming::LineReader.new(io, batch_size: batch_size) total = reader.estimate_line_count diff --git a/test/extensions/word_hash_test.rb b/test/extensions/word_hash_test.rb index e42cd4f3..53a9308a 100644 --- a/test/extensions/word_hash_test.rb +++ b/test/extensions/word_hash_test.rb @@ -12,6 +12,12 @@ def test_clean_word_hash assert_equal hash, "here are some good words of test's. I hope you love them!".clean_word_hash end + + def test_stem_to_word_hash + hash = { rubi: 'ruby', program: 'programming', eleg: 'elegance', mean: 'means', defin: 'defines' } + + assert_equal(hash, 'Ruby programming is elegant. Ruby means elegance. Elegance defines Ruby'.stem_to_word_hash) + end end class ArrayExtensionsTest < Minitest::Test diff --git a/test/keywords/cli_test.rb b/test/keywords/cli_test.rb new file mode 100644 index 00000000..64eaa5df --- /dev/null +++ b/test/keywords/cli_test.rb @@ -0,0 +1,240 @@ +# frozen_string_literal: true + +require_relative '../test_helper' +require 'classifier/keywords/cli' + +module Keywords + class CLITest < Minitest::Test + def setup + @tmpdir = Dir.mktmpdir + @model_path = File.join(@tmpdir, 'keywords.json') + end + + def teardown + FileUtils.remove_entry(@tmpdir) if @tmpdir && File.exist?(@tmpdir) + end + + def run_cli(*args, stdin: nil) + cli = Classifier::Keywords::CLI.new(args, stdin: stdin) + cli.run + end + + def make_articles + Dir.mkdir(File.join(@tmpdir, 'articles')) + File.write(File.join(@tmpdir, 'articles', 'a1.txt'), <<~TEXT) + Dogs are loyal pets + Dogs are great pets and very loyal + TEXT + File.write(File.join(@tmpdir, 'articles', 'a2.txt'), <<~TEXT) + Cats are independent and self-sufficient + Dogs are great + TEXT + File.write(File.join(@tmpdir, 'articles', 'a3.txt'), <<~TEXT) + Birds can fly and sing beautiful songs + Dogs and cats are popular pets + TEXT + end + + def make_model + tfidf = Classifier::TFIDF.new + tfidf.fit( + [ + 'Dogs are great pets and very loyal', + 'Cats are independent and self-sufficient', + 'Dogs and cats are popular pets' + ] + ) + tfidf.save_to_file(@model_path) + end + + def test_help_flag + result = run_cli('-h') + + assert_match('Usage:', result[:output]) + assert_match('Commands:', result[:output]) + assert_match('Options:', result[:output]) + assert_equal 0, result[:exit_code] + end + + def test_version_flag + result = run_cli('-v') + + assert_match(/\d+\.\d+\.\d+/, result[:output]) + assert_equal 0, result[:exit_code] + end + + def test_keywords_without_args + unless $stdin.tty? + skip( + 'This test should be skipped if the stdin contains data, ' \ + "i.e. the test is run like this: echo '' | bundle exec rake" + ) + end + result = run_cli + + assert_match('Keyword extraction and term analysis using TF-IDF', result[:output]) + assert_match('Run "keywords --help" for full usage.', result[:output]) + assert_equal 0, result[:exit_code] + end + + def test_fit_command + make_articles + result = run_cli('-m', @model_path, 'fit', File.join(@tmpdir, 'articles/*.txt')) + + assert_equal 0, result[:exit_code] + assert_predicate File.size(@model_path), :positive? + end + + def test_fit_command_stdin + result = run_cli( + '-m', @model_path, '--min-df', '2', '--max-df', '0.5', '--ngram', '1,2', + 'fit', + stdin: 'Cats are independent and self-sufficient' + ) + + assert_equal 0, result[:exit_code] + assert_predicate File.size(@model_path), :positive? + end + + def test_fit_command_with_invalid_ngram_one_value + result = run_cli( + '-m', @model_path, '--min-df', '2', '--max-df', '0.5', '--ngram', '12', + 'fit', + stdin: 'Cats are independent and self-sufficient' + ) + + assert_equal 2, result[:exit_code] + assert_match('requires exactly two values', result[:error]) + refute_path_exists @model_path + end + + def test_fit_command_with_invalid_ngram_three_value + result = run_cli( + '-m', @model_path, '--min-df', '2', '--max-df', '0.5', '--ngram', '1,2,3', + 'fit', + stdin: 'Cats are independent and self-sufficient' + ) + + assert_equal 2, result[:exit_code] + assert_match('requires exactly two values', result[:error]) + refute_path_exists @model_path + end + + def test_fit_command_with_invalid_ngram_value_not_integers + result = run_cli( + '-m', @model_path, '--min-df', '2', '--max-df', '0.5', '--ngram', '1abc,2xyz', + 'fit', + stdin: 'Cats are independent and self-sufficient' + ) + + assert_equal 2, result[:exit_code] + assert_match('must be integers', result[:error]) + refute_path_exists @model_path + end + + def test_fit_command_with_zero_ngram_bounds_error + result = run_cli( + '-m', @model_path, '--min-df', '2', '--max-df', '0.5', '--ngram', '0,1', + 'fit', + stdin: 'Cats are independent and self-sufficient' + ) + + assert_equal 2, result[:exit_code] + assert_match('bounds must be >= 1 and min <= max', result[:error]) + refute_path_exists @model_path + end + + def test_fit_command_with_ngram_min_exceeds_max_error + result = run_cli( + '-m', @model_path, '--min-df', '2', '--max-df', '0.5', '--ngram', '2,1', + 'fit', + stdin: 'Cats are independent and self-sufficient' + ) + + assert_equal 2, result[:exit_code] + assert_match('bounds must be >= 1 and min <= max', result[:error]) + refute_path_exists @model_path + end + + def test_fit_command_with_no_documents + result = run_cli('-m', @model_path, 'fit', stdin: '') + + assert_equal 2, result[:exit_code] + assert_match('No documents found to save the model', result[:error]) + refute_path_exists @model_path + end + + def test_extract_command + make_model + result = run_cli('-m', @model_path, '-n', '1', 'extract', 'Dogs and cats are great') + + assert_equal 0, result[:exit_code] + assert_match('great:0.68', result[:output]) + refute_match('cats:0.52', result[:output]) + refute_match('dogs:0.52', result[:output]) + end + + def test_extract_command_stdin + make_model + result = run_cli('-m', @model_path, '-n', '1', 'extract', stdin: 'Dogs and cats are great') + + assert_equal 0, result[:exit_code] + assert_match('great:0.68', result[:output]) + refute_match('cats:0.52', result[:output]) + refute_match('dogs:0.52', result[:output]) + end + + def test_keywords_command + make_model + result = run_cli('-m', @model_path, '-n', '1', 'Dogs and cats are great') + + assert_equal 0, result[:exit_code] + assert_match('great:0.68', result[:output]) + refute_match('cats:0.52', result[:output]) + refute_match('dogs:0.52', result[:output]) + end + + def test_keywords_command_stdin + make_model + result = run_cli('-m', @model_path, '-n', '1', stdin: 'Dogs and cats are great') + + assert_equal 0, result[:exit_code] + assert_match('great:0.68', result[:output]) + refute_match('cats:0.52', result[:output]) + refute_match('dogs:0.52', result[:output]) + end + + def test_keywords_command_output_original_words + make_model + result = run_cli('-m', @model_path, 'Dogs and cats are great. Large dog. Smart dog') + + assert_equal 0, result[:exit_code] + assert_match('great:0.38', result[:output]) + assert_match('cats:0.29', result[:output]) + assert_match('dog:0.88', result[:output]) + end + + def test_keywords_command_with_not_existing_model + result = run_cli('-m', @model_path, 'Dogs and cats are great') + + assert_equal 2, result[:exit_code] + assert_match('No model found', result[:error]) + end + + def test_keywords_command_with_negative_top_option + make_model + result = run_cli('-m', @model_path, '-n', '-2', 'Dogs and cats are great') + + assert_equal 2, result[:exit_code] + assert_match('must be positive', result[:error]) + end + + def test_info_command + make_model + result = run_cli('-m', @model_path, 'info') + + assert_equal 0, result[:exit_code] + assert_match("Documents: 3\nVocabulary: 9\nMin DF: 1\nMax DF: 1.0", result[:output]) + end + end +end diff --git a/test/streaming/multi_io_test.rb b/test/streaming/multi_io_test.rb new file mode 100644 index 00000000..76252e97 --- /dev/null +++ b/test/streaming/multi_io_test.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require_relative '../test_helper' +require 'stringio' + +class MultiIOTest < Minitest::Test + def test_each_line_returns_enumerator_without_block + stream = Classifier::Streaming::MultiIO.new( + [ + StringIO.new("11\n22\n33\n"), + StringIO.new("44\n55\n66\n"), + StringIO.new("77\n88\n99\n") + ] + ) + enum = stream.each_line + + assert_kind_of Enumerator, enum + assert_equal "11\n22\n33\n44\n55\n66\n77\n88\n99\n", enum.to_a.join + end + + def test_each_line_yields_lines_to_block + stream = Classifier::Streaming::MultiIO.new( + [ + StringIO.new("11\n22\n"), + StringIO.new("33\n44\n55\n66\n77\n88\n"), + StringIO.new("99\n") + ] + ) + lines = [] + stream.each_line do |line| + lines << line.chomp + end + + assert_equal '112233445566778899', lines.join + end +end