TestOMPTraitLexer.cpp

Go to the documentation of this file.

//===- TestOMPTraitLexer.cpp - Tests for OMP Trait Lexer -----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "kmp_traits.h"
#include "gtest/gtest.h"
 
#include <cstring>
#include <string>
 
using namespace lexer;
 
namespace {
 
//===----------------------------------------------------------------------===//
// Helpers
//===----------------------------------------------------------------------===//
 
// Compare a token's text against a C string.
static bool text_is(const token &tok, const char *expected) {
  size_t len = strlen(expected);
  if (tok.text.length() != len)
    return false;
  if (len == 0)
    return true;
  return memcmp(tok.text.begin(), expected, len) == 0;
}
 
// Lex the whole spec into a vector of tokens (excluding the trailing END).
static kmp_vector<token> tokenize(const char *spec) {
  kmp_lexer lex{kmp_str_ref(spec)};
  kmp_vector<token> tokens;
  for (token tok = lex.next(); tok.kind != token_kind::END; tok = lex.next())
    tokens.push_back(tok);
  return tokens;
}
 
// Assert a token has the expected kind and text.
static void expect_token(const token &tok, token_kind kind, const char *text) {
  EXPECT_EQ(tok.kind, kind);
  EXPECT_TRUE(text_is(tok, text))
      << "expected token text \"" << text << "\" but got \""
      << std::string(tok.text.begin(), tok.text.length()) << "\"";
}
 
//===----------------------------------------------------------------------===//
// Single Tokens
//===----------------------------------------------------------------------===//
 
TEST(LexerTest, EmptyInput) {
  kmp_lexer lex{kmp_str_ref("")};
  EXPECT_EQ(lex.next().kind, token_kind::END);
}
 
TEST(LexerTest, WhitespaceOnly) {
  kmp_lexer lex{kmp_str_ref("   \t  ")};
  EXPECT_EQ(lex.next().kind, token_kind::END);
}
 
TEST(LexerTest, EndIsSticky) {
  // Once the input is exhausted, next() keeps returning END.
  kmp_lexer lex{kmp_str_ref("*")};
  EXPECT_EQ(lex.next().kind, token_kind::STAR);
  EXPECT_EQ(lex.next().kind, token_kind::END);
  EXPECT_EQ(lex.next().kind, token_kind::END);
}
 
TEST(LexerTest, Punctuation) {
  kmp_vector<token> tokens = tokenize(",*!()[]:");
 
  ASSERT_EQ(tokens.size(), 8u);
  expect_token(tokens[0], token_kind::COMMA, ",");
  expect_token(tokens[1], token_kind::STAR, "*");
  expect_token(tokens[2], token_kind::NOT, "!");
  expect_token(tokens[3], token_kind::L_PAREN, "(");
  expect_token(tokens[4], token_kind::R_PAREN, ")");
  expect_token(tokens[5], token_kind::L_BRACKET, "[");
  expect_token(tokens[6], token_kind::R_BRACKET, "]");
  expect_token(tokens[7], token_kind::COLON, ":");
}
 
TEST(LexerTest, DoubleColon) {
  // "::" is lexed as two COLON tokens.
  kmp_vector<token> tokens = tokenize("::");
 
  ASSERT_EQ(tokens.size(), 2u);
  expect_token(tokens[0], token_kind::COLON, ":");
  expect_token(tokens[1], token_kind::COLON, ":");
}
 
TEST(LexerTest, AndOperator) {
  kmp_vector<token> tokens = tokenize("&&");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::AND, "&&");
}
 
TEST(LexerTest, OrOperator) {
  kmp_vector<token> tokens = tokenize("||");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::OR, "||");
}
 
//===----------------------------------------------------------------------===//
// Numbers
//
// The lexer does not have a dedicated number token: a run of digits is just a
// WORD. Recognizing a device number is the parser's job (see consume_clause).
//===----------------------------------------------------------------------===//
 
TEST(LexerTest, SingleDigitNumber) {
  kmp_vector<token> tokens = tokenize("0");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::WORD, "0");
}
 
TEST(LexerTest, MultiDigitNumber) {
  kmp_vector<token> tokens = tokenize("12345");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::WORD, "12345");
}
 
TEST(LexerTest, NumberList) {
  kmp_vector<token> tokens = tokenize("1,2,3");
 
  ASSERT_EQ(tokens.size(), 5u);
  expect_token(tokens[0], token_kind::WORD, "1");
  expect_token(tokens[1], token_kind::COMMA, ",");
  expect_token(tokens[2], token_kind::WORD, "2");
  expect_token(tokens[3], token_kind::COMMA, ",");
  expect_token(tokens[4], token_kind::WORD, "3");
}
 
//===----------------------------------------------------------------------===//
// Words
//===----------------------------------------------------------------------===//
 
TEST(LexerTest, SimpleWord) {
  kmp_vector<token> tokens = tokenize("uid");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::WORD, "uid");
}
 
TEST(LexerTest, WordWithDash) {
  // A uid_value may contain '-'; it stays part of the word.
  kmp_vector<token> tokens = tokenize("device-0");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::WORD, "device-0");
}
 
TEST(LexerTest, WordWithUnderscore) {
  kmp_vector<token> tokens = tokenize("my_device_123");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::WORD, "my_device_123");
}
 
TEST(LexerTest, LeadingMinusIsWord) {
  // The grammar allows a leading "-" for device numbers, but since '-' is also
  // a uid_value symbol the lexer keeps the contiguous run together as a single
  // WORD. Distinguishing a negative device number is left to the parser.
  kmp_vector<token> tokens = tokenize("-5");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::WORD, "-5");
}
 
TEST(LexerTest, WhitespaceBreaksWords) {
  // A uid_value may not contain whitespace, so spaces split the run.
  kmp_vector<token> tokens = tokenize("device - 0");
 
  ASSERT_EQ(tokens.size(), 3u);
  expect_token(tokens[0], token_kind::WORD, "device");
  expect_token(tokens[1], token_kind::WORD, "-");
  expect_token(tokens[2], token_kind::WORD, "0");
}
 
//===----------------------------------------------------------------------===//
// Full trait expressions
//===----------------------------------------------------------------------===//
 
TEST(LexerTest, UidTrait) {
  kmp_vector<token> tokens = tokenize("uid(device-0)");
 
  ASSERT_EQ(tokens.size(), 4u);
  expect_token(tokens[0], token_kind::WORD, "uid");
  expect_token(tokens[1], token_kind::L_PAREN, "(");
  expect_token(tokens[2], token_kind::WORD, "device-0");
  expect_token(tokens[3], token_kind::R_PAREN, ")");
}
 
TEST(LexerTest, NegatedUidTrait) {
  kmp_vector<token> tokens = tokenize("!uid(a)");
 
  ASSERT_EQ(tokens.size(), 5u);
  expect_token(tokens[0], token_kind::NOT, "!");
  expect_token(tokens[1], token_kind::WORD, "uid");
  expect_token(tokens[2], token_kind::L_PAREN, "(");
  expect_token(tokens[3], token_kind::WORD, "a");
  expect_token(tokens[4], token_kind::R_PAREN, ")");
}
 
TEST(LexerTest, AndGroup) {
  kmp_vector<token> tokens = tokenize("(uid(a) && uid(b))");
 
  ASSERT_EQ(tokens.size(), 11u);
  expect_token(tokens[0], token_kind::L_PAREN, "(");
  expect_token(tokens[1], token_kind::WORD, "uid");
  expect_token(tokens[2], token_kind::L_PAREN, "(");
  expect_token(tokens[3], token_kind::WORD, "a");
  expect_token(tokens[4], token_kind::R_PAREN, ")");
  expect_token(tokens[5], token_kind::AND, "&&");
  expect_token(tokens[6], token_kind::WORD, "uid");
  expect_token(tokens[7], token_kind::L_PAREN, "(");
  expect_token(tokens[8], token_kind::WORD, "b");
  expect_token(tokens[9], token_kind::R_PAREN, ")");
  expect_token(tokens[10], token_kind::R_PAREN, ")");
}
 
TEST(LexerTest, OrGroup) {
  kmp_vector<token> tokens = tokenize("uid(a)||uid(b)");
 
  ASSERT_EQ(tokens.size(), 9u);
  expect_token(tokens[0], token_kind::WORD, "uid");
  expect_token(tokens[1], token_kind::L_PAREN, "(");
  expect_token(tokens[2], token_kind::WORD, "a");
  expect_token(tokens[3], token_kind::R_PAREN, ")");
  expect_token(tokens[4], token_kind::OR, "||");
  expect_token(tokens[5], token_kind::WORD, "uid");
  expect_token(tokens[6], token_kind::L_PAREN, "(");
  expect_token(tokens[7], token_kind::WORD, "b");
  expect_token(tokens[8], token_kind::R_PAREN, ")");
}
 
TEST(LexerTest, WildcardWithLiterals) {
  kmp_vector<token> tokens = tokenize("1, *, 3");
 
  ASSERT_EQ(tokens.size(), 5u);
  expect_token(tokens[0], token_kind::WORD, "1");
  expect_token(tokens[1], token_kind::COMMA, ",");
  expect_token(tokens[2], token_kind::STAR, "*");
  expect_token(tokens[3], token_kind::COMMA, ",");
  expect_token(tokens[4], token_kind::WORD, "3");
}
 
TEST(LexerTest, IndexExpression) {
  kmp_vector<token> tokens = tokenize("[1:2:3]");
 
  ASSERT_EQ(tokens.size(), 7u);
  expect_token(tokens[0], token_kind::L_BRACKET, "[");
  expect_token(tokens[1], token_kind::WORD, "1");
  expect_token(tokens[2], token_kind::COLON, ":");
  expect_token(tokens[3], token_kind::WORD, "2");
  expect_token(tokens[4], token_kind::COLON, ":");
  expect_token(tokens[5], token_kind::WORD, "3");
  expect_token(tokens[6], token_kind::R_BRACKET, "]");
}
 
//===----------------------------------------------------------------------===//
// Whitespace handling
//===----------------------------------------------------------------------===//
 
TEST(LexerTest, LeadingAndTrailingWhitespace) {
  kmp_vector<token> tokens = tokenize("   uid ( a )   ");
 
  ASSERT_EQ(tokens.size(), 4u);
  expect_token(tokens[0], token_kind::WORD, "uid");
  expect_token(tokens[1], token_kind::L_PAREN, "(");
  expect_token(tokens[2], token_kind::WORD, "a");
  expect_token(tokens[3], token_kind::R_PAREN, ")");
}
 
TEST(LexerTest, WhitespaceAroundOperators) {
  kmp_vector<token> tokens = tokenize("uid(a)   &&   uid(b)");
 
  ASSERT_EQ(tokens.size(), 9u);
  expect_token(tokens[4], token_kind::AND, "&&");
}
 
//===----------------------------------------------------------------------===//
// Unknown / error characters
//===----------------------------------------------------------------------===//
 
TEST(LexerTest, UnknownCharacter) {
  kmp_vector<token> tokens = tokenize("@");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::UNKNOWN, "@");
}
 
TEST(LexerTest, LoneAmpersandIsUnknown) {
  kmp_vector<token> tokens = tokenize("&");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::UNKNOWN, "&");
}
 
TEST(LexerTest, LonePipeIsUnknown) {
  kmp_vector<token> tokens = tokenize("|");
 
  ASSERT_EQ(tokens.size(), 1u);
  expect_token(tokens[0], token_kind::UNKNOWN, "|");
}
 
TEST(LexerTest, TripleAmpersand) {
  // "&&&" is "&&" followed by a lone (unknown) '&'.
  kmp_vector<token> tokens = tokenize("&&&");
 
  ASSERT_EQ(tokens.size(), 2u);
  expect_token(tokens[0], token_kind::AND, "&&");
  expect_token(tokens[1], token_kind::UNKNOWN, "&");
}
 
//===----------------------------------------------------------------------===//
// peek()
//===----------------------------------------------------------------------===//
 
TEST(LexerTest, PeekDoesNotAdvance) {
  kmp_lexer lex{kmp_str_ref("uid(a)")};
 
  // Peek repeatedly returns the same token without consuming it.
  EXPECT_EQ(lex.peek().kind, token_kind::WORD);
  EXPECT_TRUE(text_is(lex.peek(), "uid"));
  EXPECT_EQ(lex.peek().kind, token_kind::WORD);
 
  // next() returns the peeked token, then advances.
  token tok = lex.next();
  expect_token(tok, token_kind::WORD, "uid");
  EXPECT_EQ(lex.peek().kind, token_kind::L_PAREN);
  expect_token(lex.next(), token_kind::L_PAREN, "(");
}
 
TEST(LexerTest, PeekAtEnd) {
  kmp_lexer lex{kmp_str_ref("")};
 
  EXPECT_EQ(lex.peek().kind, token_kind::END);
  EXPECT_EQ(lex.next().kind, token_kind::END);
  EXPECT_EQ(lex.peek().kind, token_kind::END);
}
 
} // namespace