Skip to content

Commit

Permalink
Fixed file loading
Browse files Browse the repository at this point in the history
  • Loading branch information
parzival418 committed Sep 4, 2020
1 parent 2d83461 commit 61852de
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions Encoder.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
// This file includes code which was modified from https://github.com/openai/gpt-2
const fs = require('fs')
const path = require('path');

const encoder = JSON.parse(fs.readFileSync(path.join(__dirname, './encoder.json')));
const bpe_file = fs.readFileSync(path.join(__dirname, './vocab.bpe'), 'utf-8');

const range = (x, y) => {
const res = Array.from(Array(y).keys()).slice(x)
Expand Down Expand Up @@ -61,11 +65,9 @@ function get_pairs(word) {

const pat = /'s|'t|'re|'ve|'m|'l l|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu

const encoder = JSON.parse(fs.readFileSync('./encoder.json'))
const decoder = {}
Object.keys(encoder).map(x => { decoder[encoder[x]] = x })

const bpe_file = fs.readFileSync('./vocab.bpe', 'utf-8')
const lines = bpe_file.split('\n')

// bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split("\n")[1:-1]]
Expand Down

0 comments on commit 61852de

Please sign in to comment.