towerops/test/snmpkit/snmp_lib/mib/snmp_tokenizer_test.exs

defmodule SnmpKit.SnmpLib.MIB.SnmpTokenizerTest do
  use ExUnit.Case, async: true

  alias SnmpKit.SnmpLib.MIB.SnmpTokenizer

  describe "tokenize/2" do
    test "tokenizes empty input" do
      assert {:ok, tokens} = SnmpTokenizer.tokenize(~c"", &SnmpTokenizer.null_get_line/0)
      assert [{:"$end", 1}] = tokens
    end

    test "tokenizes simple MIB header" do
      input = ~c"TEST-MIB DEFINITIONS ::= BEGIN END"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      # TEST-MIB is tokenized as :variable (uppercase identifier)
      assert :variable in token_types
      assert :DEFINITIONS in token_types
      assert :BEGIN in token_types
      assert :END in token_types
    end

    test "tokenizes reserved words" do
      reserved_words = [
        "DEFINITIONS",
        "BEGIN",
        "END",
        "IMPORTS",
        "FROM",
        "OBJECT-TYPE",
        "SYNTAX",
        "MAX-ACCESS",
        "STATUS",
        "DESCRIPTION"
      ]

      for word <- reserved_words do
        input = String.to_charlist(word)
        assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

        # First token should be the reserved word
        refute Enum.empty?(tokens)
      end
    end

    test "tokenizes identifiers" do
      input = ~c"testObject myIdentifier test123"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      atoms = Enum.filter(tokens, &match?({:atom, _, _}, &1))
      assert length(atoms) == 3
    end

    test "tokenizes numbers" do
      input = ~c"0 123 456789"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      numbers = Enum.filter(tokens, &match?({:integer, _, _}, &1))
      assert length(numbers) == 3
    end

    test "tokenizes negative numbers" do
      input = ~c"-1 -123"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      # Might be tokenized as minus sign + integer, depends on implementation
      assert length(tokens) >= 2
    end

    test "tokenizes strings with quotes" do
      input = ~c"\"test string\" \"another\""

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      # Strings might be tokenized as :string or just handled as tokens
      # Just verify tokenization succeeds
      refute Enum.empty?(tokens)
    end

    test "tokenizes special characters" do
      input = ~c"::= { } ( ) , ; ."

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      # Should have various special character tokens
      assert length(tokens) > 5
    end

    test "tokenizes OID notation" do
      input = ~c"{ test 1 2 3 }"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      # Should have braces, identifier, and numbers
      assert length(tokens) >= 6
    end

    test "tokenizes complete OBJECT-TYPE definition" do
      input = ~c"""
      testObject OBJECT-TYPE
        SYNTAX Integer32
        MAX-ACCESS read-only
        STATUS current
        DESCRIPTION "Test object"
        ::= { test 1 }
      """

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      assert :"OBJECT-TYPE" in token_types
      assert :SYNTAX in token_types
      assert :"MAX-ACCESS" in token_types
      assert :STATUS in token_types
      assert :DESCRIPTION in token_types
    end

    test "tokenizes access modifiers" do
      access_values = ["read-only", "read-write", "not-accessible", "read-create"]

      for access <- access_values do
        input = String.to_charlist(access)
        assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)
        refute Enum.empty?(tokens)
      end
    end

    test "tokenizes SMI types" do
      types = ["INTEGER", "OCTET STRING", "Counter32", "Gauge32", "TimeTicks", "IpAddress"]

      for type <- types do
        input = String.to_charlist(type)
        assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)
        refute Enum.empty?(tokens)
      end
    end

    test "tokenizes hyphenated keywords" do
      keywords = [
        "OBJECT-TYPE",
        "MAX-ACCESS",
        "MODULE-IDENTITY",
        "OBJECT-IDENTITY",
        "TEXTUAL-CONVENTION"
      ]

      for keyword <- keywords do
        input = String.to_charlist(keyword)
        assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

        # Should tokenize as single keyword token
        non_end_tokens = Enum.reject(tokens, &match?({:"$end", _}, &1))
        assert length(non_end_tokens) == 1
      end
    end

    test "handles whitespace correctly" do
      input = ~c"  test   123   \n  OBJECT-TYPE  "

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      # Whitespace should be ignored, only meaningful tokens returned
      non_end_tokens = Enum.reject(tokens, &match?({:"$end", _}, &1))
      assert length(non_end_tokens) == 3
    end

    test "handles comments" do
      input = ~c"test -- this is a comment\nOBJECT-TYPE"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      # Comments should be ignored
      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      assert :atom in token_types
      assert :"OBJECT-TYPE" in token_types
    end

    test "handles multiline input" do
      input = ~c"""
      TEST-MIB
      DEFINITIONS
      ::=
      BEGIN
      END
      """

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      # TEST-MIB is tokenized as :variable (uppercase identifier)
      assert :variable in token_types
      assert :DEFINITIONS in token_types
      assert :BEGIN in token_types
      assert :END in token_types
    end

    test "tokenizes with line numbers" do
      input = ~c"test\nDEFINITIONS\nBEGIN"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      # Tokens should have different line numbers
      lines =
        Enum.map(tokens, fn
          {_type, line} -> line
          {_type, _value, line} -> line
        end)

      assert lines |> Enum.uniq() |> length() > 1
    end

    test "preserves token order" do
      input = ~c"first second third"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      atoms = Enum.filter(tokens, &match?({:atom, _, _}, &1))

      # Should be in order
      assert length(atoms) == 3
    end

    test "handles complex MIB structure" do
      input = ~c"""
      TEST-MIB DEFINITIONS ::= BEGIN
      IMPORTS
        MODULE-IDENTITY FROM SNMPv2-SMI;

      testMIB MODULE-IDENTITY
        LAST-UPDATED "202501010000Z"
        ORGANIZATION "Test Org"
        ::= { test 1 }
      END
      """

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      assert :DEFINITIONS in token_types
      assert :BEGIN in token_types
      assert :IMPORTS in token_types
      assert :"MODULE-IDENTITY" in token_types
      assert :FROM in token_types
      assert :END in token_types
    end

    test "tokenizes SIZE constraint" do
      input = ~c"SIZE (0..255)"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      assert :SIZE in token_types
    end

    test "tokenizes SEQUENCE OF" do
      input = ~c"SEQUENCE OF TestEntry"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      assert :SEQUENCE in token_types
      assert :OF in token_types
    end
  end

  describe "format_error/1" do
    test "formats illegal character error" do
      error = {:illegal, ?@}
      formatted = SnmpTokenizer.format_error(error)

      formatted_str = List.to_string(formatted)
      assert formatted_str =~ "illegal character"
    end

    test "formats unterminated string error" do
      error = {:unterminated_string, 42}
      formatted = SnmpTokenizer.format_error(error)

      formatted_str = List.to_string(formatted)
      assert formatted_str =~ "unterminated string"
      assert formatted_str =~ "42"
    end

    test "formats unterminated quote error" do
      error = {:unterminated_quote, 10}
      formatted = SnmpTokenizer.format_error(error)

      formatted_str = List.to_string(formatted)
      assert formatted_str =~ "unterminated quote"
      assert formatted_str =~ "10"
    end

    test "formats unknown error" do
      error = {:unknown_error, "details"}
      formatted = SnmpTokenizer.format_error(error)

      refute Enum.empty?(formatted)
    end

    test "formats atom error" do
      error = :some_error
      formatted = SnmpTokenizer.format_error(error)

      refute Enum.empty?(formatted)
    end
  end

  describe "null_get_line/0" do
    test "returns eof" do
      assert SnmpTokenizer.null_get_line() == :eof
    end
  end

  describe "test/0" do
    test "runs test function without crashing" do
      # The test/0 function tokenizes a test string and prints output
      # We just verify it doesn't crash
      assert SnmpTokenizer.test() in [:ok, :error]
    end
  end

  describe "edge cases" do
    test "handles very long identifiers" do
      # Use a more reasonable length to avoid atom table limits (255 bytes max for atoms)
      long_id = "a" |> String.duplicate(200) |> String.to_charlist()

      assert {:ok, tokens} = SnmpTokenizer.tokenize(long_id, &SnmpTokenizer.null_get_line/0)
      refute Enum.empty?(tokens)
    end

    test "handles many tokens" do
      # Create input with many identifiers
      identifiers = Enum.map_join(1..100, " ", fn i -> "id#{i}" end)
      input = String.to_charlist(identifiers)

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)
      atoms = Enum.filter(tokens, &match?({:atom, _, _}, &1))
      assert length(atoms) == 100
    end

    test "handles deeply nested braces" do
      input = ~c"{ { { test } } }"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)
      assert length(tokens) > 6
    end

    test "handles mixed case keywords" do
      # Reserved words should be case-sensitive
      input = ~c"begin BEGIN"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      # Only "BEGIN" should be recognized as keyword
      begin_count = Enum.count(token_types, &(&1 == :BEGIN))
      assert begin_count == 1

      # "begin" should be an atom (lowercase identifier)
      atom_count = Enum.count(tokens, &match?({:atom, _, _}, &1))
      assert atom_count >= 1
    end

    test "handles unicode characters in strings" do
      # Basic ASCII string should work
      input = ~c"\"test\""

      assert {:ok, _tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)
    end

    test "handles empty strings" do
      input = ~c"\"\""

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)
      refute Enum.empty?(tokens)
    end

    test "handles consecutive operators" do
      input = ~c"::=::="

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)
      # Should tokenize as two ::= operators or similar
      assert length(tokens) >= 2
    end

    test "handles tabs and mixed whitespace" do
      input = ~c"test\t\tOBJECT-TYPE\n\n  BEGIN"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      assert :atom in token_types
      assert :"OBJECT-TYPE" in token_types
      assert :BEGIN in token_types
    end

    test "handles numbers with leading zeros" do
      input = ~c"001 0123"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      integers = Enum.filter(tokens, &match?({:integer, _, _}, &1))
      assert length(integers) == 2
    end

    test "handles very large numbers" do
      input = ~c"999999999999"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      integers = Enum.filter(tokens, &match?({:integer, _, _}, &1))
      assert length(integers) == 1
    end
  end

  describe "reserved word coverage" do
    test "tokenizes all common reserved words" do
      # Test a sampling of reserved words to ensure coverage
      words = [
        "DEFINITIONS",
        "BEGIN",
        "END",
        "IMPORTS",
        "FROM",
        "EXPORTS",
        "OBJECT",
        "IDENTIFIER",
        "OBJECT-TYPE",
        "SYNTAX",
        "ACCESS",
        "MAX-ACCESS",
        "STATUS",
        "DESCRIPTION",
        "REFERENCE",
        "INDEX",
        "SEQUENCE",
        "OF",
        "SIZE",
        "INTEGER",
        "OCTET",
        "STRING",
        "Counter32",
        "Gauge32",
        "MODULE-IDENTITY",
        "TEXTUAL-CONVENTION",
        "OBJECT-GROUP",
        "NOTIFICATION-TYPE"
      ]

      for word <- words do
        input = String.to_charlist(word)
        assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

        # Should successfully tokenize without error
        refute Enum.empty?(tokens)
      end
    end

    test "distinguishes reserved words from similar identifiers" do
      # "OBJECT" is reserved, "object" is lowercase atom
      input = ~c"OBJECT object"

      assert {:ok, tokens} = SnmpTokenizer.tokenize(input, &SnmpTokenizer.null_get_line/0)

      token_types =
        Enum.map(tokens, fn
          {type, _line} -> type
          {type, _value, _line} -> type
        end)

      # OBJECT should be keyword
      assert :OBJECT in token_types

      # "object" should be an atom (lowercase identifier)
      atoms = Enum.filter(tokens, &match?({:atom, _, _}, &1))
      refute Enum.empty?(atoms)
    end
  end
end