Skip to content

Instantly share code, notes, and snippets.

@JacksonTian

JacksonTian/Makefile

Last active August 29, 2015 14:07
Show Gist options
  • Save JacksonTian/7c6111831e4b03471640 to your computer and use it in GitHub Desktop.
Save JacksonTian/7c6111831e4b03471640 to your computer and use it in GitHub Desktop.
Download ZIP
V8词法分析
Raw
lexer.cpp
#include <assert.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <vector>
#include "src/v8.h"
#include "src/string-stream.h"
#include "src/scanner.h"
#include "src/scanner-character-streams.h"
using namespace v8::internal;
const byte* ReadFile(const char* name, int* length) {
FILE* file = fopen(name, "rb");
*length = 0;
if (file == NULL) return NULL;
fseek(file, 0, SEEK_END);
int file_size = ftell(file);
*length = file_size;
rewind(file);
byte* chars = new byte[file_size + 1];
for (int i = 0; i < file_size;) {
int read = static_cast<int>(fread(&chars[i], 1, file_size - i, file));
i += read;
}
fclose(file);
chars[file_size] = 0;
return chars;
}
char* substr(const byte* source, int start, int end) {
char* newstr = new char[end - start + 1];
for (int i = start; i < end; i++) {
newstr[i - start] = source[i];
}
newstr[end - start] = 0;
return newstr;
}
struct TokenWithLocation {
Token::Value value;
size_t beg;
size_t end;
TokenWithLocation() : value(Token::ILLEGAL), beg(0), end(0) { }
TokenWithLocation(Token::Value value, size_t beg, size_t end) :
value(value), beg(beg), end(end) { }
bool operator==(const TokenWithLocation& other) {
return value == other.value && beg == other.beg && end == other.end;
}
bool operator!=(const TokenWithLocation& other) {
return !(*this == other);
}
void Print(const char* prefix, const byte* content) const {
printf("%s %11s at (%d, %d) \t: %s\n",
prefix, Token::Name(value),
static_cast<int>(beg),
static_cast<int>(end),
substr(content, beg, end));
}
};
void PrintTokens(const char* name,
const std::vector<TokenWithLocation>& tokens,
const byte* content) {
printf("No of tokens: %d\n",
static_cast<int>(tokens.size()));
printf("%s:\n", name);
for (size_t i = 0; i < tokens.size(); ++i) {
tokens[i].Print("=>", content);
}
}
Token::Value Next(Scanner* scanner_, int* beg_pos, int* end_pos) {
Token::Value res = scanner_->Next();
*beg_pos = scanner_->location().beg_pos;
*end_pos = scanner_->location().end_pos;
return res;
}
int main(int argc, char* argv[]) {
if (argc < 0) {
printf("Please use it like: lexer filename.js");
return 0;
}
char* filename = argv[1];
v8::V8::Initialize();
v8::Isolate* isolate = v8::Isolate::New();
{
v8::Isolate::Scope isolate_scope(isolate);
v8::HandleScope handle_scope(isolate);
v8::Handle<v8::ObjectTemplate> global = v8::ObjectTemplate::New(isolate);
v8::Local<v8::Context> context = v8::Context::New(isolate, NULL, global);
{
v8::Context::Scope scope(context);
int length = 0;
const byte* content = ReadFile(filename, &length);
printf("%s\n", content);
UnicodeCache* unicode_cache_ = new UnicodeCache();
Scanner* scanner_ = new Scanner(unicode_cache_);
BufferedUtf16CharacterStream* stream_ = new Utf8ToUtf16CharacterStream(content, length);
scanner_->Initialize(stream_);
std::vector<TokenWithLocation> tokens;
Token::Value token;
int beg, end;
do {
token = Next(scanner_, &beg, &end);
tokens.push_back(TokenWithLocation(token, beg, end));
} while (token != Token::EOS);
PrintTokens("Baseline", tokens, content);
}
}
v8::V8::Dispose();
return 0;
}
Raw
Makefile
V8HOME = /Users/jacksontian/git/v8
FLAGS = -I$(V8HOME) \
$(V8HOME)/out/x64.release/libv8_{libbase,base,snapshot}.a \
$(V8HOME)/out/x64.release/libicu{uc,i18n,data}.a -stdlib=libstdc++
lexer:
clang++ lexer.cpp -o bin/lexer $(FLAGS)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment