chore: update node_modules with new binary files and dependencies
- Add new binary files for nodemon, onnxruntime-web, and xenova/transformers - Update various JavaScript and TypeScript files in node_modules - Remove unused files and dependencies - Add new test fixtures and documentation files
This commit is contained in:
你的名字
246
node_modules/@huggingface/jinja/src/ast.ts
generated
vendored
Normal file
246
node_modules/@huggingface/jinja/src/ast.ts
generated
vendored
Normal file
@@ -0,0 +1,246 @@
|
||||
import type { Token } from "./lexer";
|
||||
|
||||
/**
|
||||
* Statements do not result in a value at runtime. They contain one or more expressions internally.
|
||||
*/
|
||||
export class Statement {
|
||||
type = "Statement";
|
||||
}
|
||||
|
||||
/**
|
||||
* Defines a block which contains many statements. Each chat template corresponds to one Program.
|
||||
*/
|
||||
export class Program extends Statement {
|
||||
override type = "Program";
|
||||
|
||||
constructor(public body: Statement[]) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
export class If extends Statement {
|
||||
override type = "If";
|
||||
|
||||
constructor(
|
||||
public test: Expression,
|
||||
public body: Statement[],
|
||||
public alternate: Statement[]
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
export class For extends Statement {
|
||||
override type = "For";
|
||||
|
||||
constructor(
|
||||
public loopvar: Identifier | TupleLiteral,
|
||||
public iterable: Expression,
|
||||
public body: Statement[]
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
export class SetStatement extends Statement {
|
||||
override type = "Set";
|
||||
constructor(
|
||||
public assignee: Expression,
|
||||
public value: Expression
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Expressions will result in a value at runtime (unlike statements).
|
||||
*/
|
||||
export class Expression extends Statement {
|
||||
override type = "Expression";
|
||||
}
|
||||
|
||||
export class MemberExpression extends Expression {
|
||||
override type = "MemberExpression";
|
||||
|
||||
constructor(
|
||||
public object: Expression,
|
||||
public property: Expression,
|
||||
public computed: boolean
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
export class CallExpression extends Expression {
|
||||
override type = "CallExpression";
|
||||
|
||||
constructor(
|
||||
public callee: Expression,
|
||||
public args: Expression[]
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a user-defined variable or symbol in the template.
|
||||
*/
|
||||
export class Identifier extends Expression {
|
||||
override type = "Identifier";
|
||||
|
||||
/**
|
||||
* @param {string} value The name of the identifier
|
||||
*/
|
||||
constructor(public value: string) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract base class for all Literal expressions.
|
||||
* Should not be instantiated directly.
|
||||
*/
|
||||
abstract class Literal<T> extends Expression {
|
||||
override type = "Literal";
|
||||
|
||||
constructor(public value: T) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a numeric constant in the template.
|
||||
*/
|
||||
export class NumericLiteral extends Literal<number> {
|
||||
override type = "NumericLiteral";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a text constant in the template.
|
||||
*/
|
||||
export class StringLiteral extends Literal<string> {
|
||||
override type = "StringLiteral";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a boolean constant in the template.
|
||||
*/
|
||||
export class BooleanLiteral extends Literal<boolean> {
|
||||
override type = "BooleanLiteral";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents an array literal in the template.
|
||||
*/
|
||||
export class ArrayLiteral extends Literal<Expression[]> {
|
||||
override type = "ArrayLiteral";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a tuple literal in the template.
|
||||
*/
|
||||
export class TupleLiteral extends Literal<Expression[]> {
|
||||
override type = "TupleLiteral";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents an object literal in the template.
|
||||
*/
|
||||
export class ObjectLiteral extends Literal<Map<Expression, Expression>> {
|
||||
override type = "ObjectLiteral";
|
||||
}
|
||||
|
||||
/**
|
||||
* An operation with two sides, separated by an operator.
|
||||
* Note: Either side can be a Complex Expression, with order
|
||||
* of operations being determined by the operator.
|
||||
*/
|
||||
export class BinaryExpression extends Expression {
|
||||
override type = "BinaryExpression";
|
||||
|
||||
constructor(
|
||||
public operator: Token,
|
||||
public left: Expression,
|
||||
public right: Expression
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An operation with two sides, separated by the | operator.
|
||||
* Operator precedence: https://github.com/pallets/jinja/issues/379#issuecomment-168076202
|
||||
*/
|
||||
export class FilterExpression extends Expression {
|
||||
override type = "FilterExpression";
|
||||
|
||||
constructor(
|
||||
public operand: Expression,
|
||||
public filter: Identifier | CallExpression
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An operation with two sides, separated by the "is" operator.
|
||||
*/
|
||||
export class TestExpression extends Expression {
|
||||
override type = "TestExpression";
|
||||
|
||||
constructor(
|
||||
public operand: Expression,
|
||||
public negate: boolean,
|
||||
public test: Identifier // TODO: Add support for non-identifier tests
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An operation with one side (operator on the left).
|
||||
*/
|
||||
export class UnaryExpression extends Expression {
|
||||
override type = "UnaryExpression";
|
||||
|
||||
constructor(
|
||||
public operator: Token,
|
||||
public argument: Expression
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Logical negation of an expression.
|
||||
*/
|
||||
export class LogicalNegationExpression extends Expression {
|
||||
override type = "LogicalNegationExpression";
|
||||
|
||||
constructor(public argument: Expression) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
export class SliceExpression extends Expression {
|
||||
override type = "SliceExpression";
|
||||
|
||||
constructor(
|
||||
public start: Expression | undefined = undefined,
|
||||
public stop: Expression | undefined = undefined,
|
||||
public step: Expression | undefined = undefined
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
export class KeywordArgumentExpression extends Expression {
|
||||
override type = "KeywordArgumentExpression";
|
||||
|
||||
constructor(
|
||||
public key: Identifier,
|
||||
public value: Expression
|
||||
) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
58
node_modules/@huggingface/jinja/src/index.ts
generated
vendored
Normal file
58
node_modules/@huggingface/jinja/src/index.ts
generated
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* @file Jinja templating engine
|
||||
*
|
||||
* A minimalistic JavaScript reimplementation of the [Jinja](https://github.com/pallets/jinja) templating engine,
|
||||
* to support the chat templates. Special thanks to [Tyler Laceby](https://github.com/tlaceby) for his amazing
|
||||
* ["Guide to Interpreters"](https://github.com/tlaceby/guide-to-interpreters-series) tutorial series,
|
||||
* which provided the basis for this implementation.
|
||||
*
|
||||
* See the [Transformers documentation](https://huggingface.co/docs/transformers/main/en/chat_templating) for more information.
|
||||
*
|
||||
* @module index
|
||||
*/
|
||||
import { tokenize } from "./lexer";
|
||||
import { parse } from "./parser";
|
||||
import { Environment, Interpreter } from "./runtime";
|
||||
import type { Program } from "./ast";
|
||||
import type { StringValue } from "./runtime";
|
||||
import { range } from "./utils";
|
||||
|
||||
export class Template {
|
||||
parsed: Program;
|
||||
|
||||
/**
|
||||
* @param {string} template The template string
|
||||
*/
|
||||
constructor(template: string) {
|
||||
const tokens = tokenize(template, {
|
||||
lstrip_blocks: true,
|
||||
trim_blocks: true,
|
||||
});
|
||||
this.parsed = parse(tokens);
|
||||
}
|
||||
|
||||
render(items: Record<string, unknown>): string {
|
||||
// Create a new environment for this template
|
||||
const env = new Environment();
|
||||
|
||||
// Declare global variables
|
||||
env.set("false", false);
|
||||
env.set("true", true);
|
||||
env.set("raise_exception", (args: string) => {
|
||||
throw new Error(args);
|
||||
});
|
||||
env.set("range", range);
|
||||
|
||||
// Add user-defined variables
|
||||
for (const [key, value] of Object.entries(items)) {
|
||||
env.set(key, value);
|
||||
}
|
||||
|
||||
const interpreter = new Interpreter(env);
|
||||
|
||||
const result = interpreter.run(this.parsed) as StringValue;
|
||||
return result.value;
|
||||
}
|
||||
}
|
||||
|
||||
export { Environment, Interpreter, tokenize, parse };
|
||||
332
node_modules/@huggingface/jinja/src/lexer.ts
generated
vendored
Normal file
332
node_modules/@huggingface/jinja/src/lexer.ts
generated
vendored
Normal file
@@ -0,0 +1,332 @@
|
||||
/**
|
||||
* Represents tokens that our language understands in parsing.
|
||||
*/
|
||||
export const TOKEN_TYPES = Object.freeze({
|
||||
Text: "Text", // The text between Jinja statements or expressions
|
||||
|
||||
NumericLiteral: "NumericLiteral", // e.g., 123
|
||||
BooleanLiteral: "BooleanLiteral", // true or false
|
||||
StringLiteral: "StringLiteral", // 'string'
|
||||
Identifier: "Identifier", // Variables, functions, etc.
|
||||
Equals: "Equals", // =
|
||||
OpenParen: "OpenParen", // (
|
||||
CloseParen: "CloseParen", // )
|
||||
OpenStatement: "OpenStatement", // {%
|
||||
CloseStatement: "CloseStatement", // %}
|
||||
OpenExpression: "OpenExpression", // {{
|
||||
CloseExpression: "CloseExpression", // }}
|
||||
OpenSquareBracket: "OpenSquareBracket", // [
|
||||
CloseSquareBracket: "CloseSquareBracket", // ]
|
||||
OpenCurlyBracket: "OpenCurlyBracket", // {
|
||||
CloseCurlyBracket: "CloseCurlyBracket", // }
|
||||
Comma: "Comma", // ,
|
||||
Dot: "Dot", // .
|
||||
Colon: "Colon", // :
|
||||
Pipe: "Pipe", // |
|
||||
|
||||
CallOperator: "CallOperator", // ()
|
||||
AdditiveBinaryOperator: "AdditiveBinaryOperator", // + -
|
||||
MultiplicativeBinaryOperator: "MultiplicativeBinaryOperator", // * / %
|
||||
ComparisonBinaryOperator: "ComparisonBinaryOperator", // < > <= >= == !=
|
||||
UnaryOperator: "UnaryOperator", // ! - +
|
||||
|
||||
// Keywords
|
||||
Set: "Set",
|
||||
If: "If",
|
||||
For: "For",
|
||||
In: "In",
|
||||
Is: "Is",
|
||||
NotIn: "NotIn",
|
||||
Else: "Else",
|
||||
EndIf: "EndIf",
|
||||
ElseIf: "ElseIf",
|
||||
EndFor: "EndFor",
|
||||
And: "And",
|
||||
Or: "Or",
|
||||
Not: "UnaryOperator",
|
||||
});
|
||||
|
||||
export type TokenType = keyof typeof TOKEN_TYPES;
|
||||
|
||||
/**
|
||||
* Constant lookup for keywords and known identifiers + symbols.
|
||||
*/
|
||||
const KEYWORDS = Object.freeze({
|
||||
set: TOKEN_TYPES.Set,
|
||||
for: TOKEN_TYPES.For,
|
||||
in: TOKEN_TYPES.In,
|
||||
is: TOKEN_TYPES.Is,
|
||||
if: TOKEN_TYPES.If,
|
||||
else: TOKEN_TYPES.Else,
|
||||
endif: TOKEN_TYPES.EndIf,
|
||||
elif: TOKEN_TYPES.ElseIf,
|
||||
endfor: TOKEN_TYPES.EndFor,
|
||||
and: TOKEN_TYPES.And,
|
||||
or: TOKEN_TYPES.Or,
|
||||
not: TOKEN_TYPES.Not,
|
||||
"not in": TOKEN_TYPES.NotIn,
|
||||
|
||||
// Literals
|
||||
true: TOKEN_TYPES.BooleanLiteral,
|
||||
false: TOKEN_TYPES.BooleanLiteral,
|
||||
});
|
||||
|
||||
/**
|
||||
* Represents a single token in the template.
|
||||
*/
|
||||
export class Token {
|
||||
/**
|
||||
* Constructs a new Token.
|
||||
* @param {string} value The raw value as seen inside the source code.
|
||||
* @param {TokenType} type The type of token.
|
||||
*/
|
||||
constructor(
|
||||
public value: string,
|
||||
public type: TokenType
|
||||
) {}
|
||||
}
|
||||
|
||||
function isWord(char: string): boolean {
|
||||
return /\w/.test(char);
|
||||
}
|
||||
|
||||
function isInteger(char: string): boolean {
|
||||
return /[0-9]/.test(char);
|
||||
}
|
||||
|
||||
/**
|
||||
* A data structure which contains a list of rules to test
|
||||
*/
|
||||
const ORDERED_MAPPING_TABLE: [string, TokenType][] = [
|
||||
// Control sequences
|
||||
["{%", TOKEN_TYPES.OpenStatement],
|
||||
["%}", TOKEN_TYPES.CloseStatement],
|
||||
["{{", TOKEN_TYPES.OpenExpression],
|
||||
["}}", TOKEN_TYPES.CloseExpression],
|
||||
// Single character tokens
|
||||
["(", TOKEN_TYPES.OpenParen],
|
||||
[")", TOKEN_TYPES.CloseParen],
|
||||
["{", TOKEN_TYPES.OpenCurlyBracket],
|
||||
["}", TOKEN_TYPES.CloseCurlyBracket],
|
||||
["[", TOKEN_TYPES.OpenSquareBracket],
|
||||
["]", TOKEN_TYPES.CloseSquareBracket],
|
||||
[",", TOKEN_TYPES.Comma],
|
||||
[".", TOKEN_TYPES.Dot],
|
||||
[":", TOKEN_TYPES.Colon],
|
||||
["|", TOKEN_TYPES.Pipe],
|
||||
// Comparison operators
|
||||
["<=", TOKEN_TYPES.ComparisonBinaryOperator],
|
||||
[">=", TOKEN_TYPES.ComparisonBinaryOperator],
|
||||
["==", TOKEN_TYPES.ComparisonBinaryOperator],
|
||||
["!=", TOKEN_TYPES.ComparisonBinaryOperator],
|
||||
["<", TOKEN_TYPES.ComparisonBinaryOperator],
|
||||
[">", TOKEN_TYPES.ComparisonBinaryOperator],
|
||||
// Arithmetic operators
|
||||
["+", TOKEN_TYPES.AdditiveBinaryOperator],
|
||||
["-", TOKEN_TYPES.AdditiveBinaryOperator],
|
||||
["*", TOKEN_TYPES.MultiplicativeBinaryOperator],
|
||||
["/", TOKEN_TYPES.MultiplicativeBinaryOperator],
|
||||
["%", TOKEN_TYPES.MultiplicativeBinaryOperator],
|
||||
// Assignment operator
|
||||
["=", TOKEN_TYPES.Equals],
|
||||
];
|
||||
|
||||
const ESCAPE_CHARACTERS = new Map([
|
||||
["n", "\n"], // New line
|
||||
["t", "\t"], // Horizontal tab
|
||||
["r", "\r"], // Carriage return
|
||||
["b", "\b"], // Backspace
|
||||
["f", "\f"], // Form feed
|
||||
["v", "\v"], // Vertical tab
|
||||
["'", "'"], // Single quote
|
||||
['"', '"'], // Double quote
|
||||
["\\", "\\"], // Backslash
|
||||
]);
|
||||
|
||||
export interface PreprocessOptions {
|
||||
trim_blocks?: boolean;
|
||||
lstrip_blocks?: boolean;
|
||||
}
|
||||
|
||||
function preprocess(template: string, options: PreprocessOptions = {}): string {
|
||||
// According to https://jinja.palletsprojects.com/en/3.0.x/templates/#whitespace-control
|
||||
|
||||
// In the default configuration:
|
||||
// - a single trailing newline is stripped if present
|
||||
// - other whitespace (spaces, tabs, newlines etc.) is returned unchanged
|
||||
if (template.endsWith("\n")) {
|
||||
template = template.slice(0, -1);
|
||||
}
|
||||
|
||||
// Replace all comments with a placeholder
|
||||
// This ensures that comments don't interfere with the following options
|
||||
template = template.replace(/{#.*?#}/gs, "{##}");
|
||||
|
||||
if (options.lstrip_blocks) {
|
||||
// The lstrip_blocks option can also be set to strip tabs and spaces from the
|
||||
// beginning of a line to the start of a block. (Nothing will be stripped if
|
||||
// there are other characters before the start of the block.)
|
||||
template = template.replace(/^[ \t]*({[#%])/gm, "$1");
|
||||
}
|
||||
|
||||
if (options.trim_blocks) {
|
||||
// If an application configures Jinja to trim_blocks, the first newline after
|
||||
// a template tag is removed automatically (like in PHP).
|
||||
template = template.replace(/([#%]})\n/g, "$1");
|
||||
}
|
||||
|
||||
return template
|
||||
.replace(/{##}/g, "") // Remove comments
|
||||
.replace(/-%}\s*/g, "%}")
|
||||
.replace(/\s*{%-/g, "{%")
|
||||
.replace(/-}}\s*/g, "}}")
|
||||
.replace(/\s*{{-/g, "{{");
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a list of tokens from a source string.
|
||||
*/
|
||||
export function tokenize(source: string, options: PreprocessOptions = {}): Token[] {
|
||||
const tokens: Token[] = [];
|
||||
const src: string = preprocess(source, options);
|
||||
|
||||
let cursorPosition = 0;
|
||||
|
||||
const consumeWhile = (predicate: (char: string) => boolean): string => {
|
||||
let str = "";
|
||||
while (predicate(src[cursorPosition])) {
|
||||
// Check for escaped characters
|
||||
if (src[cursorPosition] === "\\") {
|
||||
// Consume the backslash
|
||||
++cursorPosition;
|
||||
// Check for end of input
|
||||
if (cursorPosition >= src.length) throw new SyntaxError("Unexpected end of input");
|
||||
|
||||
// Add the escaped character
|
||||
const escaped = src[cursorPosition++];
|
||||
const unescaped = ESCAPE_CHARACTERS.get(escaped);
|
||||
if (unescaped === undefined) {
|
||||
throw new SyntaxError(`Unexpected escaped character: ${escaped}`);
|
||||
}
|
||||
str += unescaped;
|
||||
continue;
|
||||
}
|
||||
|
||||
str += src[cursorPosition++];
|
||||
if (cursorPosition >= src.length) throw new SyntaxError("Unexpected end of input");
|
||||
}
|
||||
return str;
|
||||
};
|
||||
|
||||
// Build each token until end of input
|
||||
main: while (cursorPosition < src.length) {
|
||||
// First, consume all text that is outside of a Jinja statement or expression
|
||||
const lastTokenType = tokens.at(-1)?.type;
|
||||
if (
|
||||
lastTokenType === undefined ||
|
||||
lastTokenType === TOKEN_TYPES.CloseStatement ||
|
||||
lastTokenType === TOKEN_TYPES.CloseExpression
|
||||
) {
|
||||
let text = "";
|
||||
while (
|
||||
cursorPosition < src.length &&
|
||||
// Keep going until we hit the next Jinja statement or expression
|
||||
!(src[cursorPosition] === "{" && (src[cursorPosition + 1] === "%" || src[cursorPosition + 1] === "{"))
|
||||
) {
|
||||
// Consume text
|
||||
text += src[cursorPosition++];
|
||||
}
|
||||
|
||||
// There is some text to add
|
||||
if (text.length > 0) {
|
||||
tokens.push(new Token(text, TOKEN_TYPES.Text));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Consume (and ignore) all whitespace inside Jinja statements or expressions
|
||||
consumeWhile((char) => /\s/.test(char));
|
||||
|
||||
// Handle multi-character tokens
|
||||
const char = src[cursorPosition];
|
||||
|
||||
// Check for unary operators
|
||||
if (char === "-" || char === "+") {
|
||||
const lastTokenType = tokens.at(-1)?.type;
|
||||
if (lastTokenType === TOKEN_TYPES.Text || lastTokenType === undefined) {
|
||||
throw new SyntaxError(`Unexpected character: ${char}`);
|
||||
}
|
||||
switch (lastTokenType) {
|
||||
case TOKEN_TYPES.Identifier:
|
||||
case TOKEN_TYPES.NumericLiteral:
|
||||
case TOKEN_TYPES.BooleanLiteral:
|
||||
case TOKEN_TYPES.StringLiteral:
|
||||
case TOKEN_TYPES.CloseParen:
|
||||
case TOKEN_TYPES.CloseSquareBracket:
|
||||
// Part of a binary operator
|
||||
// a - 1, 1 - 1, true - 1, "apple" - 1, (1) - 1, a[1] - 1
|
||||
// Continue parsing normally
|
||||
break;
|
||||
|
||||
default: {
|
||||
// Is part of a unary operator
|
||||
// (-1), [-1], (1 + -1), not -1, -apple
|
||||
++cursorPosition; // consume the unary operator
|
||||
|
||||
// Check for numbers following the unary operator
|
||||
const num = consumeWhile(isInteger);
|
||||
tokens.push(
|
||||
new Token(`${char}${num}`, num.length > 0 ? TOKEN_TYPES.NumericLiteral : TOKEN_TYPES.UnaryOperator)
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to match one of the tokens in the mapping table
|
||||
for (const [char, token] of ORDERED_MAPPING_TABLE) {
|
||||
const slice = src.slice(cursorPosition, cursorPosition + char.length);
|
||||
if (slice === char) {
|
||||
tokens.push(new Token(char, token));
|
||||
cursorPosition += char.length;
|
||||
continue main;
|
||||
}
|
||||
}
|
||||
|
||||
if (char === "'" || char === '"') {
|
||||
++cursorPosition; // Skip the opening quote
|
||||
const str = consumeWhile((c) => c !== char);
|
||||
tokens.push(new Token(str, TOKEN_TYPES.StringLiteral));
|
||||
++cursorPosition; // Skip the closing quote
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isInteger(char)) {
|
||||
const num = consumeWhile(isInteger);
|
||||
tokens.push(new Token(num, TOKEN_TYPES.NumericLiteral));
|
||||
continue;
|
||||
}
|
||||
if (isWord(char)) {
|
||||
const word = consumeWhile(isWord);
|
||||
|
||||
// Check for special/reserved keywords
|
||||
// NOTE: We use Object.hasOwn() to avoid matching `.toString()` and other Object methods
|
||||
const type = Object.hasOwn(KEYWORDS, word) ? KEYWORDS[word as keyof typeof KEYWORDS] : TOKEN_TYPES.Identifier;
|
||||
|
||||
// Special case of not in:
|
||||
// If the previous token was a "not", and this token is "in"
|
||||
// then we want to combine them into a single token
|
||||
if (type === TOKEN_TYPES.In && tokens.at(-1)?.type === TOKEN_TYPES.Not) {
|
||||
tokens.pop();
|
||||
tokens.push(new Token("not in", TOKEN_TYPES.NotIn));
|
||||
} else {
|
||||
tokens.push(new Token(word, type));
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new SyntaxError(`Unexpected character: ${char}`);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
536
node_modules/@huggingface/jinja/src/parser.ts
generated
vendored
Normal file
536
node_modules/@huggingface/jinja/src/parser.ts
generated
vendored
Normal file
@@ -0,0 +1,536 @@
|
||||
import type { Token, TokenType } from "./lexer";
|
||||
import { TOKEN_TYPES } from "./lexer";
|
||||
import type { Statement } from "./ast";
|
||||
import {
|
||||
Program,
|
||||
If,
|
||||
For,
|
||||
SetStatement,
|
||||
MemberExpression,
|
||||
CallExpression,
|
||||
Identifier,
|
||||
NumericLiteral,
|
||||
StringLiteral,
|
||||
BooleanLiteral,
|
||||
ArrayLiteral,
|
||||
ObjectLiteral,
|
||||
BinaryExpression,
|
||||
FilterExpression,
|
||||
TestExpression,
|
||||
UnaryExpression,
|
||||
SliceExpression,
|
||||
KeywordArgumentExpression,
|
||||
TupleLiteral,
|
||||
} from "./ast";
|
||||
|
||||
/**
|
||||
* Generate the Abstract Syntax Tree (AST) from a list of tokens.
|
||||
* Operator precedence can be found here: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
|
||||
*/
|
||||
export function parse(tokens: Token[]): Program {
|
||||
const program = new Program([]);
|
||||
let current = 0;
|
||||
|
||||
/**
|
||||
* Consume the next token if it matches the expected type, otherwise throw an error.
|
||||
* @param type The expected token type
|
||||
* @param error The error message to throw if the token does not match the expected type
|
||||
* @returns The consumed token
|
||||
*/
|
||||
function expect(type: string, error: string): Token {
|
||||
const prev = tokens[current++];
|
||||
if (!prev || prev.type !== type) {
|
||||
throw new Error(`Parser Error: ${error}. ${prev.type} !== ${type}.`);
|
||||
}
|
||||
return prev;
|
||||
}
|
||||
|
||||
function parseAny(): Statement {
|
||||
switch (tokens[current].type) {
|
||||
case TOKEN_TYPES.Text:
|
||||
return parseText();
|
||||
case TOKEN_TYPES.OpenStatement:
|
||||
return parseJinjaStatement();
|
||||
case TOKEN_TYPES.OpenExpression:
|
||||
return parseJinjaExpression();
|
||||
default:
|
||||
throw new SyntaxError(`Unexpected token type: ${tokens[current].type}`);
|
||||
}
|
||||
}
|
||||
|
||||
function not(...types: TokenType[]): boolean {
|
||||
return current + types.length <= tokens.length && types.some((type, i) => type !== tokens[current + i].type);
|
||||
}
|
||||
|
||||
function is(...types: TokenType[]): boolean {
|
||||
return current + types.length <= tokens.length && types.every((type, i) => type === tokens[current + i].type);
|
||||
}
|
||||
|
||||
function parseText(): StringLiteral {
|
||||
return new StringLiteral(expect(TOKEN_TYPES.Text, "Expected text token").value);
|
||||
}
|
||||
|
||||
function parseJinjaStatement(): Statement {
|
||||
// Consume {% %} tokens
|
||||
expect(TOKEN_TYPES.OpenStatement, "Expected opening statement token");
|
||||
|
||||
let result;
|
||||
switch (tokens[current].type) {
|
||||
case TOKEN_TYPES.Set:
|
||||
++current;
|
||||
result = parseSetStatement();
|
||||
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
|
||||
break;
|
||||
|
||||
case TOKEN_TYPES.If:
|
||||
++current;
|
||||
result = parseIfStatement();
|
||||
expect(TOKEN_TYPES.OpenStatement, "Expected {% token");
|
||||
expect(TOKEN_TYPES.EndIf, "Expected endif token");
|
||||
expect(TOKEN_TYPES.CloseStatement, "Expected %} token");
|
||||
break;
|
||||
|
||||
case TOKEN_TYPES.For:
|
||||
++current;
|
||||
result = parseForStatement();
|
||||
expect(TOKEN_TYPES.OpenStatement, "Expected {% token");
|
||||
expect(TOKEN_TYPES.EndFor, "Expected endfor token");
|
||||
expect(TOKEN_TYPES.CloseStatement, "Expected %} token");
|
||||
break;
|
||||
default:
|
||||
throw new SyntaxError(`Unknown statement type: ${tokens[current].type}`);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseJinjaExpression(): Statement {
|
||||
// Consume {{ }} tokens
|
||||
expect(TOKEN_TYPES.OpenExpression, "Expected opening expression token");
|
||||
|
||||
const result = parseExpression();
|
||||
|
||||
expect(TOKEN_TYPES.CloseExpression, "Expected closing expression token");
|
||||
return result;
|
||||
}
|
||||
|
||||
// NOTE: `set` acts as both declaration statement and assignment expression
|
||||
function parseSetStatement(): Statement {
|
||||
const left = parseExpression();
|
||||
|
||||
if (is(TOKEN_TYPES.Equals)) {
|
||||
++current;
|
||||
const value = parseSetStatement();
|
||||
|
||||
return new SetStatement(left, value);
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
function parseIfStatement(): If {
|
||||
const test = parseExpression();
|
||||
|
||||
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
|
||||
|
||||
const body: Statement[] = [];
|
||||
const alternate: Statement[] = [];
|
||||
|
||||
// Keep parsing if body until we reach the first {% elif %} or {% else %} or {% endif %}
|
||||
while (
|
||||
!(
|
||||
tokens[current]?.type === TOKEN_TYPES.OpenStatement &&
|
||||
(tokens[current + 1]?.type === TOKEN_TYPES.ElseIf ||
|
||||
tokens[current + 1]?.type === TOKEN_TYPES.Else ||
|
||||
tokens[current + 1]?.type === TOKEN_TYPES.EndIf)
|
||||
)
|
||||
) {
|
||||
body.push(parseAny());
|
||||
}
|
||||
|
||||
// Alternate branch: Check for {% elif %} or {% else %}
|
||||
if (
|
||||
tokens[current]?.type === TOKEN_TYPES.OpenStatement &&
|
||||
tokens[current + 1]?.type !== TOKEN_TYPES.EndIf // There is some body
|
||||
) {
|
||||
++current; // eat {% token
|
||||
if (is(TOKEN_TYPES.ElseIf)) {
|
||||
expect(TOKEN_TYPES.ElseIf, "Expected elseif token");
|
||||
alternate.push(parseIfStatement());
|
||||
} else {
|
||||
// tokens[current]?.type === TokenType.Else
|
||||
expect(TOKEN_TYPES.Else, "Expected else token");
|
||||
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
|
||||
|
||||
// keep going until we hit {% endif %}
|
||||
while (
|
||||
!(tokens[current]?.type === TOKEN_TYPES.OpenStatement && tokens[current + 1]?.type === TOKEN_TYPES.EndIf)
|
||||
) {
|
||||
alternate.push(parseAny());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new If(test, body, alternate);
|
||||
}
|
||||
|
||||
function parseExpressionSequence(primary = false): Statement {
|
||||
const fn = primary ? parsePrimaryExpression : parseExpression;
|
||||
const expressions = [fn()];
|
||||
const isTuple = is(TOKEN_TYPES.Comma);
|
||||
while (isTuple) {
|
||||
++current; // consume comma
|
||||
expressions.push(fn());
|
||||
if (!is(TOKEN_TYPES.Comma)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return isTuple ? new TupleLiteral(expressions) : expressions[0];
|
||||
}
|
||||
|
||||
function parseForStatement(): For {
|
||||
// e.g., `message` in `for message in messages`
|
||||
const loopVariable = parseExpressionSequence(true); // should be an identifier
|
||||
if (!(loopVariable instanceof Identifier || loopVariable instanceof TupleLiteral)) {
|
||||
throw new SyntaxError(`Expected identifier/tuple for the loop variable, got ${loopVariable.type} instead`);
|
||||
}
|
||||
|
||||
expect(TOKEN_TYPES.In, "Expected `in` keyword following loop variable");
|
||||
|
||||
// `messages` in `for message in messages`
|
||||
const iterable = parseExpression();
|
||||
|
||||
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
|
||||
|
||||
// Body of for loop
|
||||
const body: Statement[] = [];
|
||||
|
||||
// Keep going until we hit {% endfor
|
||||
while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndFor)) {
|
||||
body.push(parseAny());
|
||||
}
|
||||
|
||||
return new For(loopVariable, iterable, body);
|
||||
}
|
||||
|
||||
function parseExpression(): Statement {
|
||||
// Choose parse function with lowest precedence
|
||||
return parseTernaryExpression();
|
||||
}
|
||||
|
||||
function parseTernaryExpression(): Statement {
|
||||
const a = parseLogicalOrExpression();
|
||||
if (is(TOKEN_TYPES.If)) {
|
||||
// Ternary expression
|
||||
++current; // consume if
|
||||
const predicate = parseLogicalOrExpression();
|
||||
expect(TOKEN_TYPES.Else, "Expected else token");
|
||||
const b = parseLogicalOrExpression();
|
||||
return new If(predicate, [a], [b]);
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
function parseLogicalOrExpression(): Statement {
|
||||
let left = parseLogicalAndExpression();
|
||||
while (is(TOKEN_TYPES.Or)) {
|
||||
const operator = tokens[current];
|
||||
++current;
|
||||
const right = parseLogicalAndExpression();
|
||||
left = new BinaryExpression(operator, left, right);
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
function parseLogicalAndExpression(): Statement {
|
||||
let left = parseLogicalNegationExpression();
|
||||
while (is(TOKEN_TYPES.And)) {
|
||||
const operator = tokens[current];
|
||||
++current;
|
||||
const right = parseLogicalNegationExpression();
|
||||
left = new BinaryExpression(operator, left, right);
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
function parseLogicalNegationExpression(): Statement {
|
||||
let right: UnaryExpression | undefined;
|
||||
|
||||
// Try parse unary operators
|
||||
while (is(TOKEN_TYPES.Not)) {
|
||||
// not not ...
|
||||
const operator = tokens[current];
|
||||
++current;
|
||||
const arg = parseLogicalNegationExpression(); // not test.x === not (test.x)
|
||||
right = new UnaryExpression(operator, arg);
|
||||
}
|
||||
|
||||
return right ?? parseComparisonExpression();
|
||||
}
|
||||
|
||||
function parseComparisonExpression(): Statement {
|
||||
// NOTE: membership has same precedence as comparison
|
||||
// e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in ('apple' == ('b' in 'banana')))
|
||||
let left = parseAdditiveExpression();
|
||||
while (is(TOKEN_TYPES.ComparisonBinaryOperator) || is(TOKEN_TYPES.In) || is(TOKEN_TYPES.NotIn)) {
|
||||
const operator = tokens[current];
|
||||
++current;
|
||||
const right = parseAdditiveExpression();
|
||||
left = new BinaryExpression(operator, left, right);
|
||||
}
|
||||
return left;
|
||||
}
|
||||
function parseAdditiveExpression(): Statement {
|
||||
let left = parseMultiplicativeExpression();
|
||||
while (is(TOKEN_TYPES.AdditiveBinaryOperator)) {
|
||||
const operator = tokens[current];
|
||||
++current;
|
||||
const right = parseMultiplicativeExpression();
|
||||
left = new BinaryExpression(operator, left, right);
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
function parseCallMemberExpression(): Statement {
|
||||
// Handle member expressions recursively
|
||||
|
||||
const member = parseMemberExpression(); // foo.x
|
||||
|
||||
if (is(TOKEN_TYPES.OpenParen)) {
|
||||
// foo.x()
|
||||
return parseCallExpression(member);
|
||||
}
|
||||
return member;
|
||||
}
|
||||
|
||||
function parseCallExpression(callee: Statement): CallExpression {
|
||||
let callExpression = new CallExpression(callee, parseArgs());
|
||||
|
||||
if (is(TOKEN_TYPES.OpenParen)) {
|
||||
// foo.x()()
|
||||
callExpression = parseCallExpression(callExpression);
|
||||
}
|
||||
|
||||
return callExpression;
|
||||
}
|
||||
|
||||
function parseArgs(): Statement[] {
|
||||
// add (x + 5, foo())
|
||||
expect(TOKEN_TYPES.OpenParen, "Expected opening parenthesis for arguments list");
|
||||
|
||||
const args = parseArgumentsList();
|
||||
|
||||
expect(TOKEN_TYPES.CloseParen, "Expected closing parenthesis for arguments list");
|
||||
return args;
|
||||
}
|
||||
function parseArgumentsList(): Statement[] {
|
||||
// comma-separated arguments list
|
||||
|
||||
const args = [];
|
||||
while (!is(TOKEN_TYPES.CloseParen)) {
|
||||
let argument = parseExpression();
|
||||
|
||||
if (is(TOKEN_TYPES.Equals)) {
|
||||
// keyword argument
|
||||
// e.g., func(x = 5, y = a or b)
|
||||
++current; // consume equals
|
||||
if (!(argument instanceof Identifier)) {
|
||||
throw new SyntaxError(`Expected identifier for keyword argument`);
|
||||
}
|
||||
const value = parseExpression();
|
||||
argument = new KeywordArgumentExpression(argument, value);
|
||||
}
|
||||
args.push(argument);
|
||||
if (is(TOKEN_TYPES.Comma)) {
|
||||
++current; // consume comma
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function parseMemberExpressionArgumentsList(): Statement {
|
||||
// NOTE: This also handles slice expressions colon-separated arguments list
|
||||
// e.g., ['test'], [0], [:2], [1:], [1:2], [1:2:3]
|
||||
|
||||
const slices: (Statement | undefined)[] = [];
|
||||
let isSlice = false;
|
||||
while (!is(TOKEN_TYPES.CloseSquareBracket)) {
|
||||
if (is(TOKEN_TYPES.Colon)) {
|
||||
// A case where a default is used
|
||||
// e.g., [:2] will be parsed as [undefined, 2]
|
||||
slices.push(undefined);
|
||||
++current; // consume colon
|
||||
isSlice = true;
|
||||
} else {
|
||||
slices.push(parseExpression());
|
||||
if (is(TOKEN_TYPES.Colon)) {
|
||||
++current; // consume colon after expression, if it exists
|
||||
isSlice = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (slices.length === 0) {
|
||||
// []
|
||||
throw new SyntaxError(`Expected at least one argument for member/slice expression`);
|
||||
}
|
||||
|
||||
if (isSlice) {
|
||||
if (slices.length > 3) {
|
||||
throw new SyntaxError(`Expected 0-3 arguments for slice expression`);
|
||||
}
|
||||
return new SliceExpression(...slices);
|
||||
}
|
||||
|
||||
return slices[0] as Statement; // normal member expression
|
||||
}
|
||||
|
||||
function parseMemberExpression(): Statement {
|
||||
let object = parsePrimaryExpression();
|
||||
|
||||
while (is(TOKEN_TYPES.Dot) || is(TOKEN_TYPES.OpenSquareBracket)) {
|
||||
const operator = tokens[current]; // . or [
|
||||
++current;
|
||||
let property: Statement;
|
||||
const computed = operator.type !== TOKEN_TYPES.Dot;
|
||||
if (computed) {
|
||||
// computed (i.e., bracket notation: obj[expr])
|
||||
property = parseMemberExpressionArgumentsList();
|
||||
expect(TOKEN_TYPES.CloseSquareBracket, "Expected closing square bracket");
|
||||
} else {
|
||||
// non-computed (i.e., dot notation: obj.expr)
|
||||
property = parsePrimaryExpression(); // should be an identifier
|
||||
if (property.type !== "Identifier") {
|
||||
throw new SyntaxError(`Expected identifier following dot operator`);
|
||||
}
|
||||
}
|
||||
object = new MemberExpression(object, property, computed);
|
||||
}
|
||||
return object;
|
||||
}
|
||||
|
||||
function parseMultiplicativeExpression(): Statement {
|
||||
let left = parseTestExpression();
|
||||
|
||||
// Multiplicative operators have higher precedence than test expressions
|
||||
// e.g., (4 * 4 is divisibleby(2)) evaluates as (4 * (4 is divisibleby(2)))
|
||||
|
||||
while (is(TOKEN_TYPES.MultiplicativeBinaryOperator)) {
|
||||
const operator = tokens[current];
|
||||
++current;
|
||||
const right = parseTestExpression();
|
||||
left = new BinaryExpression(operator, left, right);
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
function parseTestExpression(): Statement {
|
||||
let operand = parseFilterExpression();
|
||||
|
||||
while (is(TOKEN_TYPES.Is)) {
|
||||
// Support chaining tests
|
||||
++current; // consume is
|
||||
const negate = is(TOKEN_TYPES.Not);
|
||||
if (negate) {
|
||||
++current; // consume not
|
||||
}
|
||||
|
||||
let filter = parsePrimaryExpression();
|
||||
if (filter instanceof BooleanLiteral) {
|
||||
// Special case: treat boolean literals as identifiers
|
||||
filter = new Identifier(filter.value.toString());
|
||||
}
|
||||
if (!(filter instanceof Identifier)) {
|
||||
throw new SyntaxError(`Expected identifier for the test`);
|
||||
}
|
||||
// TODO: Add support for non-identifier tests
|
||||
operand = new TestExpression(operand, negate, filter);
|
||||
}
|
||||
return operand;
|
||||
}
|
||||
|
||||
function parseFilterExpression(): Statement {
|
||||
let operand = parseCallMemberExpression();
|
||||
|
||||
while (is(TOKEN_TYPES.Pipe)) {
|
||||
// Support chaining filters
|
||||
++current; // consume pipe
|
||||
let filter = parsePrimaryExpression(); // should be an identifier
|
||||
if (!(filter instanceof Identifier)) {
|
||||
throw new SyntaxError(`Expected identifier for the filter`);
|
||||
}
|
||||
if (is(TOKEN_TYPES.OpenParen)) {
|
||||
filter = parseCallExpression(filter);
|
||||
}
|
||||
operand = new FilterExpression(operand, filter as Identifier | CallExpression);
|
||||
}
|
||||
return operand;
|
||||
}
|
||||
|
||||
function parsePrimaryExpression(): Statement {
|
||||
// Primary expression: number, string, identifier, function call, parenthesized expression
|
||||
const token = tokens[current];
|
||||
switch (token.type) {
|
||||
case TOKEN_TYPES.NumericLiteral:
|
||||
++current;
|
||||
return new NumericLiteral(Number(token.value));
|
||||
case TOKEN_TYPES.StringLiteral:
|
||||
++current;
|
||||
return new StringLiteral(token.value);
|
||||
case TOKEN_TYPES.BooleanLiteral:
|
||||
++current;
|
||||
return new BooleanLiteral(token.value === "true");
|
||||
case TOKEN_TYPES.Identifier:
|
||||
++current;
|
||||
return new Identifier(token.value);
|
||||
case TOKEN_TYPES.OpenParen: {
|
||||
++current; // consume opening parenthesis
|
||||
const expression = parseExpressionSequence();
|
||||
if (tokens[current].type !== TOKEN_TYPES.CloseParen) {
|
||||
throw new SyntaxError(`Expected closing parenthesis, got ${tokens[current].type} instead`);
|
||||
}
|
||||
++current; // consume closing parenthesis
|
||||
return expression;
|
||||
}
|
||||
case TOKEN_TYPES.OpenSquareBracket: {
|
||||
++current; // consume opening square bracket
|
||||
|
||||
const values = [];
|
||||
while (!is(TOKEN_TYPES.CloseSquareBracket)) {
|
||||
values.push(parseExpression());
|
||||
|
||||
if (is(TOKEN_TYPES.Comma)) {
|
||||
++current; // consume comma
|
||||
}
|
||||
}
|
||||
++current; // consume closing square bracket
|
||||
|
||||
return new ArrayLiteral(values);
|
||||
}
|
||||
case TOKEN_TYPES.OpenCurlyBracket: {
|
||||
++current; // consume opening curly bracket
|
||||
|
||||
const values = new Map();
|
||||
while (!is(TOKEN_TYPES.CloseCurlyBracket)) {
|
||||
const key = parseExpression();
|
||||
expect(TOKEN_TYPES.Colon, "Expected colon between key and value in object literal");
|
||||
const value = parseExpression();
|
||||
values.set(key, value);
|
||||
|
||||
if (is(TOKEN_TYPES.Comma)) {
|
||||
++current; // consume comma
|
||||
}
|
||||
}
|
||||
++current; // consume closing curly bracket
|
||||
|
||||
return new ObjectLiteral(values);
|
||||
}
|
||||
default:
|
||||
throw new SyntaxError(`Unexpected token: ${token.type}`);
|
||||
}
|
||||
}
|
||||
|
||||
while (current < tokens.length) {
|
||||
program.body.push(parseAny());
|
||||
}
|
||||
|
||||
return program;
|
||||
}
|
||||
918
node_modules/@huggingface/jinja/src/runtime.ts
generated
vendored
Normal file
918
node_modules/@huggingface/jinja/src/runtime.ts
generated
vendored
Normal file
@@ -0,0 +1,918 @@
|
||||
import type {
|
||||
NumericLiteral,
|
||||
StringLiteral,
|
||||
BooleanLiteral,
|
||||
ArrayLiteral,
|
||||
Statement,
|
||||
Program,
|
||||
If,
|
||||
For,
|
||||
SetStatement,
|
||||
MemberExpression,
|
||||
CallExpression,
|
||||
Identifier,
|
||||
BinaryExpression,
|
||||
FilterExpression,
|
||||
TestExpression,
|
||||
UnaryExpression,
|
||||
SliceExpression,
|
||||
KeywordArgumentExpression,
|
||||
ObjectLiteral,
|
||||
TupleLiteral,
|
||||
} from "./ast";
|
||||
import { slice, titleCase } from "./utils";
|
||||
|
||||
export type AnyRuntimeValue =
|
||||
| NumericValue
|
||||
| StringValue
|
||||
| BooleanValue
|
||||
| ObjectValue
|
||||
| ArrayValue
|
||||
| FunctionValue
|
||||
| NullValue
|
||||
| UndefinedValue;
|
||||
|
||||
/**
|
||||
* Abstract base class for all Runtime values.
|
||||
* Should not be instantiated directly.
|
||||
*/
|
||||
abstract class RuntimeValue<T> {
|
||||
type = "RuntimeValue";
|
||||
value: T;
|
||||
|
||||
/**
|
||||
* A collection of built-in functions for this type.
|
||||
*/
|
||||
builtins = new Map<string, AnyRuntimeValue>();
|
||||
|
||||
/**
|
||||
* Creates a new RuntimeValue.
|
||||
*/
|
||||
constructor(value: T = undefined as unknown as T) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines truthiness or falsiness of the runtime value.
|
||||
* This function should be overridden by subclasses if it has custom truthiness criteria.
|
||||
* @returns {BooleanValue} BooleanValue(true) if the value is truthy, BooleanValue(false) otherwise.
|
||||
*/
|
||||
__bool__(): BooleanValue {
|
||||
return new BooleanValue(!!this.value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a numeric value at runtime.
|
||||
*/
|
||||
export class NumericValue extends RuntimeValue<number> {
|
||||
override type = "NumericValue";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a string value at runtime.
|
||||
*/
|
||||
export class StringValue extends RuntimeValue<string> {
|
||||
override type = "StringValue";
|
||||
|
||||
override builtins = new Map<string, AnyRuntimeValue>([
|
||||
[
|
||||
"upper",
|
||||
new FunctionValue(() => {
|
||||
return new StringValue(this.value.toUpperCase());
|
||||
}),
|
||||
],
|
||||
[
|
||||
"lower",
|
||||
new FunctionValue(() => {
|
||||
return new StringValue(this.value.toLowerCase());
|
||||
}),
|
||||
],
|
||||
[
|
||||
"strip",
|
||||
new FunctionValue(() => {
|
||||
return new StringValue(this.value.trim());
|
||||
}),
|
||||
],
|
||||
[
|
||||
"title",
|
||||
new FunctionValue(() => {
|
||||
return new StringValue(titleCase(this.value));
|
||||
}),
|
||||
],
|
||||
["length", new NumericValue(this.value.length)],
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a boolean value at runtime.
|
||||
*/
|
||||
export class BooleanValue extends RuntimeValue<boolean> {
|
||||
override type = "BooleanValue";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents an Object value at runtime.
|
||||
*/
|
||||
export class ObjectValue extends RuntimeValue<Map<string, AnyRuntimeValue>> {
|
||||
override type = "ObjectValue";
|
||||
|
||||
/**
|
||||
* NOTE: necessary to override since all JavaScript arrays are considered truthy,
|
||||
* while only non-empty Python arrays are consider truthy.
|
||||
*
|
||||
* e.g.,
|
||||
* - JavaScript: {} && 5 -> 5
|
||||
* - Python: {} and 5 -> {}
|
||||
*/
|
||||
override __bool__(): BooleanValue {
|
||||
return new BooleanValue(this.value.size > 0);
|
||||
}
|
||||
|
||||
override builtins: Map<string, AnyRuntimeValue> = new Map<string, AnyRuntimeValue>([
|
||||
[
|
||||
"get",
|
||||
new FunctionValue(([key, defaultValue]) => {
|
||||
if (!(key instanceof StringValue)) {
|
||||
throw new Error(`Object key must be a string: got ${key.type}`);
|
||||
}
|
||||
return this.value.get(key.value) ?? defaultValue ?? new NullValue();
|
||||
}),
|
||||
],
|
||||
[
|
||||
"items",
|
||||
new FunctionValue(() => {
|
||||
return new ArrayValue(
|
||||
Array.from(this.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value]))
|
||||
);
|
||||
}),
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents an Array value at runtime.
|
||||
*/
|
||||
export class ArrayValue extends RuntimeValue<AnyRuntimeValue[]> {
|
||||
override type = "ArrayValue";
|
||||
override builtins = new Map<string, AnyRuntimeValue>([["length", new NumericValue(this.value.length)]]);
|
||||
|
||||
/**
|
||||
* NOTE: necessary to override since all JavaScript arrays are considered truthy,
|
||||
* while only non-empty Python arrays are consider truthy.
|
||||
*
|
||||
* e.g.,
|
||||
* - JavaScript: [] && 5 -> 5
|
||||
* - Python: [] and 5 -> []
|
||||
*/
|
||||
override __bool__(): BooleanValue {
|
||||
return new BooleanValue(this.value.length > 0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a Tuple value at runtime.
|
||||
* NOTE: We extend ArrayValue since JavaScript does not have a built-in Tuple type.
|
||||
*/
|
||||
export class TupleValue extends ArrayValue {
|
||||
override type = "TupleValue";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a Function value at runtime.
|
||||
*/
|
||||
export class FunctionValue extends RuntimeValue<(args: AnyRuntimeValue[], scope: Environment) => AnyRuntimeValue> {
|
||||
override type = "FunctionValue";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a Null value at runtime.
|
||||
*/
|
||||
export class NullValue extends RuntimeValue<null> {
|
||||
override type = "NullValue";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents an Undefined value at runtime.
|
||||
*/
|
||||
export class UndefinedValue extends RuntimeValue<undefined> {
|
||||
override type = "UndefinedValue";
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents the current environment (scope) at runtime.
|
||||
*/
|
||||
export class Environment {
|
||||
/**
|
||||
* The variables declared in this environment.
|
||||
*/
|
||||
variables: Map<string, AnyRuntimeValue> = new Map([
|
||||
[
|
||||
"namespace",
|
||||
new FunctionValue((args) => {
|
||||
if (args.length === 0) {
|
||||
return new ObjectValue(new Map());
|
||||
}
|
||||
if (args.length !== 1 || !(args[0] instanceof ObjectValue)) {
|
||||
throw new Error("`namespace` expects either zero arguments or a single object argument");
|
||||
}
|
||||
return args[0];
|
||||
}),
|
||||
],
|
||||
]);
|
||||
|
||||
/**
|
||||
* The tests available in this environment.
|
||||
*/
|
||||
tests: Map<string, (...value: AnyRuntimeValue[]) => boolean> = new Map([
|
||||
["boolean", (operand) => operand.type === "BooleanValue"],
|
||||
["callable", (operand) => operand instanceof FunctionValue],
|
||||
[
|
||||
"odd",
|
||||
(operand) => {
|
||||
if (operand.type !== "NumericValue") {
|
||||
throw new Error(`Cannot apply test "odd" to type: ${operand.type}`);
|
||||
}
|
||||
return (operand as NumericValue).value % 2 !== 0;
|
||||
},
|
||||
],
|
||||
[
|
||||
"even",
|
||||
(operand) => {
|
||||
if (operand.type !== "NumericValue") {
|
||||
throw new Error(`Cannot apply test "even" to type: ${operand.type}`);
|
||||
}
|
||||
return (operand as NumericValue).value % 2 === 0;
|
||||
},
|
||||
],
|
||||
["false", (operand) => operand.type === "BooleanValue" && !(operand as BooleanValue).value],
|
||||
["true", (operand) => operand.type === "BooleanValue" && (operand as BooleanValue).value],
|
||||
["number", (operand) => operand.type === "NumericValue"],
|
||||
["integer", (operand) => operand.type === "NumericValue" && Number.isInteger((operand as NumericValue).value)],
|
||||
["iterable", (operand) => operand instanceof ArrayValue || operand instanceof StringValue],
|
||||
[
|
||||
"lower",
|
||||
(operand) => {
|
||||
const str = (operand as StringValue).value;
|
||||
return operand.type === "StringValue" && str === str.toLowerCase();
|
||||
},
|
||||
],
|
||||
[
|
||||
"upper",
|
||||
(operand) => {
|
||||
const str = (operand as StringValue).value;
|
||||
return operand.type === "StringValue" && str === str.toUpperCase();
|
||||
},
|
||||
],
|
||||
["none", (operand) => operand.type === "NullValue"],
|
||||
["defined", (operand) => operand.type !== "UndefinedValue"],
|
||||
["undefined", (operand) => operand.type === "UndefinedValue"],
|
||||
["equalto", (a, b) => a.value === b.value],
|
||||
]);
|
||||
|
||||
constructor(public parent?: Environment) {}
|
||||
|
||||
/**
|
||||
* Set the value of a variable in the current environment.
|
||||
*/
|
||||
set(name: string, value: unknown): AnyRuntimeValue {
|
||||
return this.declareVariable(name, convertToRuntimeValues(value));
|
||||
}
|
||||
|
||||
private declareVariable(name: string, value: AnyRuntimeValue): AnyRuntimeValue {
|
||||
if (this.variables.has(name)) {
|
||||
throw new SyntaxError(`Variable already declared: ${name}`);
|
||||
}
|
||||
this.variables.set(name, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
// private assignVariable(name: string, value: AnyRuntimeValue): AnyRuntimeValue {
|
||||
// const env = this.resolve(name);
|
||||
// env.variables.set(name, value);
|
||||
// return value;
|
||||
// }
|
||||
|
||||
/**
|
||||
* Set variable in the current scope.
|
||||
* See https://jinja.palletsprojects.com/en/3.0.x/templates/#assignments for more information.
|
||||
*/
|
||||
setVariable(name: string, value: AnyRuntimeValue): AnyRuntimeValue {
|
||||
this.variables.set(name, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the environment in which the variable is declared.
|
||||
* @param {string} name The name of the variable.
|
||||
* @returns {Environment} The environment in which the variable is declared.
|
||||
*/
|
||||
private resolve(name: string): Environment {
|
||||
if (this.variables.has(name)) {
|
||||
return this;
|
||||
}
|
||||
|
||||
// Traverse scope chain
|
||||
if (this.parent) {
|
||||
return this.parent.resolve(name);
|
||||
}
|
||||
|
||||
throw new Error(`Unknown variable: ${name}`);
|
||||
}
|
||||
|
||||
lookupVariable(name: string): AnyRuntimeValue {
|
||||
try {
|
||||
return this.resolve(name).variables.get(name) ?? new UndefinedValue();
|
||||
} catch {
|
||||
return new UndefinedValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class Interpreter {
|
||||
global: Environment;
|
||||
|
||||
constructor(env?: Environment) {
|
||||
this.global = env ?? new Environment();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the program.
|
||||
*/
|
||||
run(program: Program): AnyRuntimeValue {
|
||||
return this.evaluate(program, this.global);
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates expressions following the binary operation type.
|
||||
*/
|
||||
private evaluateBinaryExpression(node: BinaryExpression, environment: Environment): AnyRuntimeValue {
|
||||
const left = this.evaluate(node.left, environment);
|
||||
|
||||
// Logical operators
|
||||
// NOTE: Short-circuiting is handled by the `evaluate` function
|
||||
switch (node.operator.value) {
|
||||
case "and":
|
||||
return left.__bool__().value ? this.evaluate(node.right, environment) : left;
|
||||
case "or":
|
||||
return left.__bool__().value ? left : this.evaluate(node.right, environment);
|
||||
}
|
||||
|
||||
// Equality operators
|
||||
const right = this.evaluate(node.right, environment);
|
||||
switch (node.operator.value) {
|
||||
case "==":
|
||||
return new BooleanValue(left.value == right.value);
|
||||
case "!=":
|
||||
return new BooleanValue(left.value != right.value);
|
||||
}
|
||||
|
||||
if (left instanceof UndefinedValue || right instanceof UndefinedValue) {
|
||||
throw new Error("Cannot perform operation on undefined values");
|
||||
} else if (left instanceof NullValue || right instanceof NullValue) {
|
||||
throw new Error("Cannot perform operation on null values");
|
||||
} else if (left instanceof NumericValue && right instanceof NumericValue) {
|
||||
// Evaulate pure numeric operations with binary operators.
|
||||
switch (node.operator.value) {
|
||||
// Arithmetic operators
|
||||
case "+":
|
||||
return new NumericValue(left.value + right.value);
|
||||
case "-":
|
||||
return new NumericValue(left.value - right.value);
|
||||
case "*":
|
||||
return new NumericValue(left.value * right.value);
|
||||
case "/":
|
||||
return new NumericValue(left.value / right.value);
|
||||
case "%":
|
||||
return new NumericValue(left.value % right.value);
|
||||
|
||||
// Comparison operators
|
||||
case "<":
|
||||
return new BooleanValue(left.value < right.value);
|
||||
case ">":
|
||||
return new BooleanValue(left.value > right.value);
|
||||
case ">=":
|
||||
return new BooleanValue(left.value >= right.value);
|
||||
case "<=":
|
||||
return new BooleanValue(left.value <= right.value);
|
||||
}
|
||||
} else if (left instanceof ArrayValue && right instanceof ArrayValue) {
|
||||
// Evaluate array operands with binary operator.
|
||||
switch (node.operator.value) {
|
||||
case "+":
|
||||
return new ArrayValue(left.value.concat(right.value));
|
||||
}
|
||||
} else if (right instanceof ArrayValue) {
|
||||
const member = right.value.find((x) => x.value === left.value) !== undefined;
|
||||
switch (node.operator.value) {
|
||||
case "in":
|
||||
return new BooleanValue(member);
|
||||
case "not in":
|
||||
return new BooleanValue(!member);
|
||||
}
|
||||
}
|
||||
|
||||
if (left instanceof StringValue || right instanceof StringValue) {
|
||||
// Support string concatenation as long as at least one operand is a string
|
||||
switch (node.operator.value) {
|
||||
case "+":
|
||||
return new StringValue(left.value.toString() + right.value.toString());
|
||||
}
|
||||
}
|
||||
|
||||
if (left instanceof StringValue && right instanceof StringValue) {
|
||||
switch (node.operator.value) {
|
||||
case "in":
|
||||
return new BooleanValue(right.value.includes(left.value));
|
||||
case "not in":
|
||||
return new BooleanValue(!right.value.includes(left.value));
|
||||
}
|
||||
}
|
||||
|
||||
if (left instanceof StringValue && right instanceof ObjectValue) {
|
||||
switch (node.operator.value) {
|
||||
case "in":
|
||||
return new BooleanValue(right.value.has(left.value));
|
||||
case "not in":
|
||||
return new BooleanValue(!right.value.has(left.value));
|
||||
}
|
||||
}
|
||||
|
||||
throw new SyntaxError(`Unknown operator "${node.operator.value}" between ${left.type} and ${right.type}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates expressions following the filter operation type.
|
||||
*/
|
||||
private evaluateFilterExpression(node: FilterExpression, environment: Environment): AnyRuntimeValue {
|
||||
const operand = this.evaluate(node.operand, environment);
|
||||
|
||||
// For now, we only support the built-in filters
|
||||
// TODO: Add support for non-identifier filters
|
||||
// e.g., functions which return filters: {{ numbers | select("odd") }}
|
||||
// TODO: Add support for user-defined filters
|
||||
// const filter = environment.lookupVariable(node.filter.value);
|
||||
// if (!(filter instanceof FunctionValue)) {
|
||||
// throw new Error(`Filter must be a function: got ${filter.type}`);
|
||||
// }
|
||||
// return filter.value([operand], environment);
|
||||
|
||||
// https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-filters
|
||||
|
||||
if (node.filter.type === "Identifier") {
|
||||
const filter = node.filter as Identifier;
|
||||
|
||||
if (operand instanceof ArrayValue) {
|
||||
switch (filter.value) {
|
||||
case "list":
|
||||
return operand;
|
||||
case "first":
|
||||
return operand.value[0];
|
||||
case "last":
|
||||
return operand.value[operand.value.length - 1];
|
||||
case "length":
|
||||
return new NumericValue(operand.value.length);
|
||||
case "reverse":
|
||||
return new ArrayValue(operand.value.reverse());
|
||||
case "sort":
|
||||
return new ArrayValue(
|
||||
operand.value.sort((a, b) => {
|
||||
if (a.type !== b.type) {
|
||||
throw new Error(`Cannot compare different types: ${a.type} and ${b.type}`);
|
||||
}
|
||||
switch (a.type) {
|
||||
case "NumericValue":
|
||||
return (a as NumericValue).value - (b as NumericValue).value;
|
||||
case "StringValue":
|
||||
return (a as StringValue).value.localeCompare((b as StringValue).value);
|
||||
default:
|
||||
throw new Error(`Cannot compare type: ${a.type}`);
|
||||
}
|
||||
})
|
||||
);
|
||||
default:
|
||||
throw new Error(`Unknown ArrayValue filter: ${filter.value}`);
|
||||
}
|
||||
} else if (operand instanceof StringValue) {
|
||||
switch (filter.value) {
|
||||
case "length":
|
||||
return new NumericValue(operand.value.length);
|
||||
case "upper":
|
||||
return new StringValue(operand.value.toUpperCase());
|
||||
case "lower":
|
||||
return new StringValue(operand.value.toLowerCase());
|
||||
case "title":
|
||||
return new StringValue(titleCase(operand.value));
|
||||
case "capitalize":
|
||||
return new StringValue(operand.value.charAt(0).toUpperCase() + operand.value.slice(1));
|
||||
case "trim":
|
||||
return new StringValue(operand.value.trim());
|
||||
default:
|
||||
throw new Error(`Unknown StringValue filter: ${filter.value}`);
|
||||
}
|
||||
} else if (operand instanceof NumericValue) {
|
||||
switch (filter.value) {
|
||||
case "abs":
|
||||
return new NumericValue(Math.abs(operand.value));
|
||||
default:
|
||||
throw new Error(`Unknown NumericValue filter: ${filter.value}`);
|
||||
}
|
||||
} else if (operand instanceof ObjectValue) {
|
||||
switch (filter.value) {
|
||||
case "items":
|
||||
return new ArrayValue(
|
||||
Array.from(operand.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value]))
|
||||
);
|
||||
case "length":
|
||||
return new NumericValue(operand.value.size);
|
||||
default:
|
||||
throw new Error(`Unknown ObjectValue filter: ${filter.value}`);
|
||||
}
|
||||
}
|
||||
throw new Error(`Cannot apply filter "${filter.value}" to type: ${operand.type}`);
|
||||
} else if (node.filter.type === "CallExpression") {
|
||||
const filter = node.filter as CallExpression;
|
||||
|
||||
if (filter.callee.type !== "Identifier") {
|
||||
throw new Error(`Unknown filter: ${filter.callee.type}`);
|
||||
}
|
||||
const filterName = (filter.callee as Identifier).value;
|
||||
|
||||
if (operand instanceof ArrayValue) {
|
||||
switch (filterName) {
|
||||
case "selectattr": {
|
||||
if (operand.value.some((x) => !(x instanceof ObjectValue))) {
|
||||
throw new Error("`selectattr` can only be applied to array of objects");
|
||||
}
|
||||
if (filter.args.some((x) => x.type !== "StringLiteral")) {
|
||||
throw new Error("arguments of `selectattr` must be strings");
|
||||
}
|
||||
|
||||
const [attr, testName, value] = filter.args.map((x) => this.evaluate(x, environment)) as StringValue[];
|
||||
|
||||
let testFunction: (...x: AnyRuntimeValue[]) => boolean;
|
||||
if (testName) {
|
||||
// Get the test function from the environment
|
||||
const test = environment.tests.get(testName.value);
|
||||
if (!test) {
|
||||
throw new Error(`Unknown test: ${testName.value}`);
|
||||
}
|
||||
testFunction = test;
|
||||
} else {
|
||||
// Default to truthiness of first argument
|
||||
testFunction = (...x: AnyRuntimeValue[]) => x[0].__bool__().value;
|
||||
}
|
||||
|
||||
// Filter the array using the test function
|
||||
const filtered = (operand.value as ObjectValue[]).filter((item) => {
|
||||
const a = item.value.get(attr.value);
|
||||
if (a) {
|
||||
return testFunction(a, value);
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
return new ArrayValue(filtered);
|
||||
}
|
||||
}
|
||||
throw new Error(`Unknown ArrayValue filter: ${filterName}`);
|
||||
} else {
|
||||
throw new Error(`Cannot apply filter "${filterName}" to type: ${operand.type}`);
|
||||
}
|
||||
}
|
||||
throw new Error(`Unknown filter: ${node.filter.type}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates expressions following the test operation type.
|
||||
*/
|
||||
private evaluateTestExpression(node: TestExpression, environment: Environment): BooleanValue {
|
||||
// For now, we only support the built-in tests
|
||||
// https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-tests
|
||||
//
|
||||
// TODO: Add support for non-identifier tests. e.g., divisibleby(number)
|
||||
const operand = this.evaluate(node.operand, environment);
|
||||
|
||||
const test = environment.tests.get(node.test.value);
|
||||
if (!test) {
|
||||
throw new Error(`Unknown test: ${node.test.value}`);
|
||||
}
|
||||
const result = test(operand);
|
||||
return new BooleanValue(node.negate ? !result : result);
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluates expressions following the unary operation type.
|
||||
*/
|
||||
private evaluateUnaryExpression(node: UnaryExpression, environment: Environment): AnyRuntimeValue {
|
||||
const argument = this.evaluate(node.argument, environment);
|
||||
|
||||
switch (node.operator.value) {
|
||||
case "not":
|
||||
return new BooleanValue(!argument.value);
|
||||
default:
|
||||
throw new SyntaxError(`Unknown operator: ${node.operator.value}`);
|
||||
}
|
||||
}
|
||||
|
||||
private evalProgram(program: Program, environment: Environment): StringValue {
|
||||
return this.evaluateBlock(program.body, environment);
|
||||
}
|
||||
|
||||
private evaluateBlock(statements: Statement[], environment: Environment): StringValue {
|
||||
// Jinja templates always evaluate to a String,
|
||||
// so we accumulate the result of each statement into a final string
|
||||
|
||||
let result = "";
|
||||
for (const statement of statements) {
|
||||
const lastEvaluated = this.evaluate(statement, environment);
|
||||
|
||||
if (lastEvaluated.type !== "NullValue" && lastEvaluated.type !== "UndefinedValue") {
|
||||
result += lastEvaluated.value;
|
||||
}
|
||||
}
|
||||
|
||||
return new StringValue(result);
|
||||
}
|
||||
|
||||
private evaluateIdentifier(node: Identifier, environment: Environment): AnyRuntimeValue {
|
||||
return environment.lookupVariable(node.value);
|
||||
}
|
||||
|
||||
private evaluateCallExpression(expr: CallExpression, environment: Environment): AnyRuntimeValue {
|
||||
// Accumulate all keyword arguments into a single object, which will be
|
||||
// used as the final argument in the call function.
|
||||
const args: AnyRuntimeValue[] = [];
|
||||
const kwargs = new Map();
|
||||
for (const argument of expr.args) {
|
||||
if (argument.type === "KeywordArgumentExpression") {
|
||||
const kwarg = argument as KeywordArgumentExpression;
|
||||
kwargs.set(kwarg.key.value, this.evaluate(kwarg.value, environment));
|
||||
} else {
|
||||
args.push(this.evaluate(argument, environment));
|
||||
}
|
||||
}
|
||||
if (kwargs.size > 0) {
|
||||
args.push(new ObjectValue(kwargs));
|
||||
}
|
||||
|
||||
const fn = this.evaluate(expr.callee, environment);
|
||||
if (fn.type !== "FunctionValue") {
|
||||
throw new Error(`Cannot call something that is not a function: got ${fn.type}`);
|
||||
}
|
||||
return (fn as FunctionValue).value(args, environment);
|
||||
}
|
||||
|
||||
private evaluateSliceExpression(
|
||||
object: AnyRuntimeValue,
|
||||
expr: SliceExpression,
|
||||
environment: Environment
|
||||
): ArrayValue | StringValue {
|
||||
if (!(object instanceof ArrayValue || object instanceof StringValue)) {
|
||||
throw new Error("Slice object must be an array or string");
|
||||
}
|
||||
|
||||
const start = this.evaluate(expr.start, environment);
|
||||
const stop = this.evaluate(expr.stop, environment);
|
||||
const step = this.evaluate(expr.step, environment);
|
||||
|
||||
// Validate arguments
|
||||
if (!(start instanceof NumericValue || start instanceof UndefinedValue)) {
|
||||
throw new Error("Slice start must be numeric or undefined");
|
||||
}
|
||||
if (!(stop instanceof NumericValue || stop instanceof UndefinedValue)) {
|
||||
throw new Error("Slice stop must be numeric or undefined");
|
||||
}
|
||||
if (!(step instanceof NumericValue || step instanceof UndefinedValue)) {
|
||||
throw new Error("Slice step must be numeric or undefined");
|
||||
}
|
||||
|
||||
if (object instanceof ArrayValue) {
|
||||
return new ArrayValue(slice(object.value, start.value, stop.value, step.value));
|
||||
} else {
|
||||
return new StringValue(slice(Array.from(object.value), start.value, stop.value, step.value).join(""));
|
||||
}
|
||||
}
|
||||
|
||||
private evaluateMemberExpression(expr: MemberExpression, environment: Environment): AnyRuntimeValue {
|
||||
const object = this.evaluate(expr.object, environment);
|
||||
|
||||
let property;
|
||||
if (expr.computed) {
|
||||
if (expr.property.type === "SliceExpression") {
|
||||
return this.evaluateSliceExpression(object, expr.property as SliceExpression, environment);
|
||||
} else {
|
||||
property = this.evaluate(expr.property, environment);
|
||||
}
|
||||
} else {
|
||||
property = new StringValue((expr.property as Identifier).value);
|
||||
}
|
||||
|
||||
let value;
|
||||
if (object instanceof ObjectValue) {
|
||||
if (!(property instanceof StringValue)) {
|
||||
throw new Error(`Cannot access property with non-string: got ${property.type}`);
|
||||
}
|
||||
value = object.value.get(property.value) ?? object.builtins.get(property.value);
|
||||
} else if (object instanceof ArrayValue || object instanceof StringValue) {
|
||||
if (property instanceof NumericValue) {
|
||||
value = object.value.at(property.value);
|
||||
if (object instanceof StringValue) {
|
||||
value = new StringValue(object.value.at(property.value));
|
||||
}
|
||||
} else if (property instanceof StringValue) {
|
||||
value = object.builtins.get(property.value);
|
||||
} else {
|
||||
throw new Error(`Cannot access property with non-string/non-number: got ${property.type}`);
|
||||
}
|
||||
} else {
|
||||
if (!(property instanceof StringValue)) {
|
||||
throw new Error(`Cannot access property with non-string: got ${property.type}`);
|
||||
}
|
||||
value = object.builtins.get(property.value);
|
||||
}
|
||||
|
||||
return value instanceof RuntimeValue ? value : new UndefinedValue();
|
||||
}
|
||||
|
||||
private evaluateSet(node: SetStatement, environment: Environment): NullValue {
|
||||
const rhs = this.evaluate(node.value, environment);
|
||||
if (node.assignee.type === "Identifier") {
|
||||
const variableName = (node.assignee as Identifier).value;
|
||||
environment.setVariable(variableName, rhs);
|
||||
} else if (node.assignee.type === "MemberExpression") {
|
||||
const member = node.assignee as MemberExpression;
|
||||
|
||||
const object = this.evaluate(member.object, environment);
|
||||
if (!(object instanceof ObjectValue)) {
|
||||
throw new Error("Cannot assign to member of non-object");
|
||||
}
|
||||
if (member.property.type !== "Identifier") {
|
||||
throw new Error("Cannot assign to member with non-identifier property");
|
||||
}
|
||||
object.value.set((member.property as Identifier).value, rhs);
|
||||
} else {
|
||||
throw new Error(`Invalid LHS inside assignment expression: ${JSON.stringify(node.assignee)}`);
|
||||
}
|
||||
|
||||
return new NullValue();
|
||||
}
|
||||
|
||||
private evaluateIf(node: If, environment: Environment): StringValue {
|
||||
const test = this.evaluate(node.test, environment);
|
||||
return this.evaluateBlock(test.__bool__().value ? node.body : node.alternate, environment);
|
||||
}
|
||||
|
||||
private evaluateFor(node: For, environment: Environment): StringValue {
|
||||
// Scope for the for loop
|
||||
const scope = new Environment(environment);
|
||||
|
||||
const iterable = this.evaluate(node.iterable, scope);
|
||||
if (!(iterable instanceof ArrayValue)) {
|
||||
throw new Error(`Expected iterable type in for loop: got ${iterable.type}`);
|
||||
}
|
||||
|
||||
let result = "";
|
||||
|
||||
for (let i = 0; i < iterable.value.length; ++i) {
|
||||
// Update the loop variable
|
||||
// TODO: Only create object once, then update value?
|
||||
const loop = new Map([
|
||||
["index", new NumericValue(i + 1)],
|
||||
["index0", new NumericValue(i)],
|
||||
["revindex", new NumericValue(iterable.value.length - i)],
|
||||
["revindex0", new NumericValue(iterable.value.length - i - 1)],
|
||||
["first", new BooleanValue(i === 0)],
|
||||
["last", new BooleanValue(i === iterable.value.length - 1)],
|
||||
["length", new NumericValue(iterable.value.length)],
|
||||
["previtem", i > 0 ? iterable.value[i - 1] : new UndefinedValue()],
|
||||
["nextitem", i < iterable.value.length - 1 ? iterable.value[i + 1] : new UndefinedValue()],
|
||||
] as [string, AnyRuntimeValue][]);
|
||||
|
||||
scope.setVariable("loop", new ObjectValue(loop));
|
||||
|
||||
const current = iterable.value[i];
|
||||
|
||||
// For this iteration, set the loop variable to the current element
|
||||
if (node.loopvar.type === "Identifier") {
|
||||
scope.setVariable((node.loopvar as Identifier).value, current);
|
||||
} else if (node.loopvar.type === "TupleLiteral") {
|
||||
const loopvar = node.loopvar as TupleLiteral;
|
||||
if (current.type !== "ArrayValue") {
|
||||
throw new Error(`Cannot unpack non-iterable type: ${current.type}`);
|
||||
}
|
||||
const c = current as ArrayValue;
|
||||
|
||||
// check if too few or many items to unpack
|
||||
if (loopvar.value.length !== c.value.length) {
|
||||
throw new Error(`Too ${loopvar.value.length > c.value.length ? "few" : "many"} items to unpack`);
|
||||
}
|
||||
for (let j = 0; j < loopvar.value.length; ++j) {
|
||||
if (loopvar.value[j].type !== "Identifier") {
|
||||
throw new Error(`Cannot unpack non-identifier type: ${loopvar.value[j].type}`);
|
||||
}
|
||||
scope.setVariable((loopvar.value[j] as Identifier).value, c.value[j]);
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluate the body of the for loop
|
||||
const evaluated = this.evaluateBlock(node.body, scope);
|
||||
result += evaluated.value;
|
||||
}
|
||||
|
||||
return new StringValue(result);
|
||||
}
|
||||
|
||||
evaluate(statement: Statement | undefined, environment: Environment): AnyRuntimeValue {
|
||||
if (statement === undefined) return new UndefinedValue();
|
||||
|
||||
switch (statement.type) {
|
||||
// Program
|
||||
case "Program":
|
||||
return this.evalProgram(statement as Program, environment);
|
||||
|
||||
// Statements
|
||||
case "Set":
|
||||
return this.evaluateSet(statement as SetStatement, environment);
|
||||
case "If":
|
||||
return this.evaluateIf(statement as If, environment);
|
||||
case "For":
|
||||
return this.evaluateFor(statement as For, environment);
|
||||
|
||||
// Expressions
|
||||
case "NumericLiteral":
|
||||
return new NumericValue(Number((statement as NumericLiteral).value));
|
||||
case "StringLiteral":
|
||||
return new StringValue((statement as StringLiteral).value);
|
||||
case "BooleanLiteral":
|
||||
return new BooleanValue((statement as BooleanLiteral).value);
|
||||
case "ArrayLiteral":
|
||||
return new ArrayValue((statement as ArrayLiteral).value.map((x) => this.evaluate(x, environment)));
|
||||
case "TupleLiteral":
|
||||
return new TupleValue((statement as TupleLiteral).value.map((x) => this.evaluate(x, environment)));
|
||||
case "ObjectLiteral": {
|
||||
const mapping = new Map();
|
||||
for (const [key, value] of (statement as ObjectLiteral).value) {
|
||||
const evaluatedKey = this.evaluate(key, environment);
|
||||
if (!(evaluatedKey instanceof StringValue)) {
|
||||
throw new Error(`Object keys must be strings: got ${evaluatedKey.type}`);
|
||||
}
|
||||
mapping.set(evaluatedKey.value, this.evaluate(value, environment));
|
||||
}
|
||||
return new ObjectValue(mapping);
|
||||
}
|
||||
case "Identifier":
|
||||
return this.evaluateIdentifier(statement as Identifier, environment);
|
||||
case "CallExpression":
|
||||
return this.evaluateCallExpression(statement as CallExpression, environment);
|
||||
case "MemberExpression":
|
||||
return this.evaluateMemberExpression(statement as MemberExpression, environment);
|
||||
|
||||
case "UnaryExpression":
|
||||
return this.evaluateUnaryExpression(statement as UnaryExpression, environment);
|
||||
case "BinaryExpression":
|
||||
return this.evaluateBinaryExpression(statement as BinaryExpression, environment);
|
||||
case "FilterExpression":
|
||||
return this.evaluateFilterExpression(statement as FilterExpression, environment);
|
||||
case "TestExpression":
|
||||
return this.evaluateTestExpression(statement as TestExpression, environment);
|
||||
|
||||
default:
|
||||
throw new SyntaxError(`Unknown node type: ${statement.type}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to convert JavaScript values to runtime values.
|
||||
*/
|
||||
function convertToRuntimeValues(input: unknown): AnyRuntimeValue {
|
||||
switch (typeof input) {
|
||||
case "number":
|
||||
return new NumericValue(input);
|
||||
case "string":
|
||||
return new StringValue(input);
|
||||
case "boolean":
|
||||
return new BooleanValue(input);
|
||||
case "object":
|
||||
if (input === null) {
|
||||
return new NullValue();
|
||||
} else if (Array.isArray(input)) {
|
||||
return new ArrayValue(input.map(convertToRuntimeValues));
|
||||
} else {
|
||||
return new ObjectValue(
|
||||
new Map(Object.entries(input).map(([key, value]) => [key, convertToRuntimeValues(value)]))
|
||||
);
|
||||
}
|
||||
case "function":
|
||||
// Wrap the user's function in a runtime function
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
return new FunctionValue((args, _scope) => {
|
||||
// NOTE: `_scope` is not used since it's in the global scope
|
||||
const result = input(...args.map((x) => x.value)) ?? null; // map undefined -> null
|
||||
return convertToRuntimeValues(result);
|
||||
});
|
||||
default:
|
||||
throw new Error(`Cannot convert to runtime value: ${input}`);
|
||||
}
|
||||
}
|
||||
54
node_modules/@huggingface/jinja/src/utils.ts
generated
vendored
Normal file
54
node_modules/@huggingface/jinja/src/utils.ts
generated
vendored
Normal file
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* Function that mimics Python's range() function.
|
||||
* @param start The start value of the range.
|
||||
* @param stop The stop value of the range. If not provided, start will be 0 and stop will be the provided start value.
|
||||
* @param step The step value of the range. Defaults to 1.
|
||||
* @returns The range of numbers.
|
||||
*/
|
||||
export function range(start: number, stop?: number, step = 1): number[] {
|
||||
if (stop === undefined) {
|
||||
stop = start;
|
||||
start = 0;
|
||||
}
|
||||
|
||||
const result: number[] = [];
|
||||
for (let i = start; i < stop; i += step) {
|
||||
result.push(i);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Function that mimics Python's array slicing.
|
||||
* @param array The array to slice.
|
||||
* @param start The start index of the slice. Defaults to 0.
|
||||
* @param stop The last index of the slice. Defaults to `array.length`.
|
||||
* @param step The step value of the slice. Defaults to 1.
|
||||
* @returns The sliced array.
|
||||
*/
|
||||
export function slice<T>(array: T[], start?: number, stop?: number, step = 1): T[] {
|
||||
const direction = Math.sign(step);
|
||||
|
||||
if (direction >= 0) {
|
||||
start = (start ??= 0) < 0 ? Math.max(array.length + start, 0) : Math.min(start, array.length);
|
||||
stop = (stop ??= array.length) < 0 ? Math.max(array.length + stop, 0) : Math.min(stop, array.length);
|
||||
} else {
|
||||
start = (start ??= array.length - 1) < 0 ? Math.max(array.length + start, -1) : Math.min(start, array.length - 1);
|
||||
stop = (stop ??= -1) < -1 ? Math.max(array.length + stop, -1) : Math.min(stop, array.length - 1);
|
||||
}
|
||||
|
||||
const result: T[] = [];
|
||||
for (let i = start; direction * i < direction * stop; i += step) {
|
||||
result.push(array[i]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Function that mimics Python's string.title() function.
|
||||
* @param value The string to title case.
|
||||
* @returns The title cased string.
|
||||
*/
|
||||
export function titleCase(value: string): string {
|
||||
return value.replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
}
|
||||
Reference in New Issue
Block a user