Skip to content

Commit

Permalink
Initial commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@zseymour
zseymour committed Jul 29, 2014
1 parent c34adb7 commit f66f4cb
Show file tree
Hide file tree
Showing 4 changed files with 23,054 additions and 0 deletions.
137 changes: 137 additions & 0 deletions compute-accuracy.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <malloc.h>
#include <ctype.h>

const long long max_size = 2000; // max length of strings
const long long N = 1; // number of closest words
const long long max_w = 50; // max length of vocabulary entries

int main(int argc, char **argv)
{
FILE *f;
char st1[max_size], st2[max_size], st3[max_size], st4[max_size], bestw[N][max_size], file_name[max_size], ch;
float dist, len, bestd[N], vec[max_size];
long long words, size, a, b, c, d, b1, b2, b3, threshold = 0;
float *M;
char *vocab;
int TCN, CCN = 0, TACN = 0, CACN = 0, SECN = 0, SYCN = 0, SEAC = 0, SYAC = 0, QID = 0, TQ = 0, TQS = 0;
if (argc < 2) {
printf("Usage: ./compute-accuracy <FILE> <threshold>\nwhere FILE contains word projections, and threshold is used to reduce vocabulary of the model for fast approximate evaluation (0 = off, otherwise typical value is 30000)\n");
return 0;
}
strcpy(file_name, argv[1]);
if (argc > 2) threshold = atoi(argv[2]);
f = fopen(file_name, "rb");
if (f == NULL) {
printf("Input file not found\n");
return -1;
}
fscanf(f, "%lld", &words);
if (threshold) if (words > threshold) words = threshold;
fscanf(f, "%lld", &size);
vocab = (char *)malloc(words * max_w * sizeof(char));
M = (float *)malloc(words * size * sizeof(float));
if (M == NULL) {
printf("Cannot allocate memory: %lld MB\n", words * size * sizeof(float) / 1048576);
return -1;
}
for (b = 0; b < words; b++) {
fscanf(f, "%s%c", &vocab[b * max_w], &ch);
for (a = 0; a < max_w; a++) vocab[b * max_w + a] = toupper(vocab[b * max_w + a]);
for (a = 0; a < size; a++) fread(&M[a + b * size], sizeof(float), 1, f);
len = 0;
for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
len = sqrt(len);
for (a = 0; a < size; a++) M[a + b * size] /= len;
}
fclose(f);
TCN = 0;
while (1) {
for (a = 0; a < N; a++) bestd[a] = 0;
for (a = 0; a < N; a++) bestw[a][0] = 0;
scanf("%s", st1);
for (a = 0; a < strlen(st1); a++) st1[a] = toupper(st1[a]);
if ((!strcmp(st1, ":")) || (!strcmp(st1, "EXIT")) || feof(stdin)) {
if (TCN == 0) TCN = 1;
if (QID != 0) {
printf("ACCURACY TOP1: %.2f %% (%d / %d)\n", CCN / (float)TCN * 100, CCN, TCN);
printf("Total accuracy: %.2f %% Semantic accuracy: %.2f %% Syntactic accuracy: %.2f %% \n", CACN / (float)TACN * 100, SEAC / (float)SECN * 100, SYAC / (float)SYCN * 100);
}
QID++;
scanf("%s", st1);
if (feof(stdin)) break;
printf("%s:\n", st1);
TCN = 0;
CCN = 0;
continue;
}
if (!strcmp(st1, "EXIT")) break;
scanf("%s", st2);
for (a = 0; a < strlen(st2); a++) st2[a] = toupper(st2[a]);
scanf("%s", st3);
for (a = 0; a<strlen(st3); a++) st3[a] = toupper(st3[a]);
scanf("%s", st4);
for (a = 0; a < strlen(st4); a++) st4[a] = toupper(st4[a]);
for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st1)) break;
b1 = b;
for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st2)) break;
b2 = b;
for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st3)) break;
b3 = b;
for (a = 0; a < N; a++) bestd[a] = 0;
for (a = 0; a < N; a++) bestw[a][0] = 0;
TQ++;
if (b1 == words) continue;
if (b2 == words) continue;
if (b3 == words) continue;
for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st4)) break;
if (b == words) continue;
for (a = 0; a < size; a++) vec[a] = (M[a + b2 * size] - M[a + b1 * size]) + M[a + b3 * size];
TQS++;
for (c = 0; c < words; c++) {
if (c == b1) continue;
if (c == b2) continue;
if (c == b3) continue;
dist = 0;
for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
for (a = 0; a < N; a++) {
if (dist > bestd[a]) {
for (d = N - 1; d > a; d--) {
bestd[d] = bestd[d - 1];
strcpy(bestw[d], bestw[d - 1]);
}
bestd[a] = dist;
strcpy(bestw[a], &vocab[c * max_w]);
break;
}
}
}
if (!strcmp(st4, bestw[0])) {
CCN++;
CACN++;
if (QID <= 5) SEAC++; else SYAC++;
}
if (QID <= 5) SECN++; else SYCN++;
TCN++;
TACN++;
}
printf("Questions seen / total: %d %d %.2f %% \n", TQS, TQ, TQS/(float)TQ*100);
return 0;
}
136 changes: 136 additions & 0 deletions distance.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <stdio.h>
#include <string.h>
#include <math.h>
#include <malloc.h>

const long long max_size = 2000; // max length of strings
const long long N = 40; // number of closest words that will be shown
const long long max_w = 50; // max length of vocabulary entries

int main(int argc, char **argv) {
FILE *f;
char st1[max_size];
char *bestw[N];
char file_name[max_size], st[100][max_size];
float dist, len, bestd[N], vec[max_size];
long long words, size, a, b, c, d, cn, bi[100];
char ch;
float *M;
char *vocab;
if (argc < 2) {
printf("Usage: ./distance <FILE>\nwhere FILE contains word projections in the BINARY FORMAT\n");
return 0;
}
strcpy(file_name, argv[1]);
f = fopen(file_name, "rb");
if (f == NULL) {
printf("Input file not found\n");
return -1;
}
fscanf(f, "%lld", &words);
fscanf(f, "%lld", &size);
vocab = (char *)malloc((long long)words * max_w * sizeof(char));
for (a = 0; a < N; a++) bestw[a] = (char *)malloc(max_size * sizeof(char));
M = (float *)malloc((long long)words * (long long)size * sizeof(float));
if (M == NULL) {
printf("Cannot allocate memory: %lld MB %lld %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
return -1;
}
for (b = 0; b < words; b++) {
fscanf(f, "%s%c", &vocab[b * max_w], &ch);
for (a = 0; a < size; a++) fread(&M[a + b * size], sizeof(float), 1, f);
len = 0;
for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
len = sqrt(len);
for (a = 0; a < size; a++) M[a + b * size] /= len;
}
fclose(f);
while (1) {
for (a = 0; a < N; a++) bestd[a] = 0;
for (a = 0; a < N; a++) bestw[a][0] = 0;
printf("Enter word or sentence (EXIT to break): ");
a = 0;
while (1) {
st1[a] = fgetc(stdin);
if ((st1[a] == '\n') || (a >= max_size - 1)) {
st1[a] = 0;
break;
}
a++;
}
if (!strcmp(st1, "EXIT")) break;
cn = 0;
b = 0;
c = 0;
while (1) {
st[cn][b] = st1[c];
b++;
c++;
st[cn][b] = 0;
if (st1[c] == 0) break;
if (st1[c] == ' ') {
cn++;
b = 0;
c++;
}
}
cn++;
for (a = 0; a < cn; a++) {
for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
if (b == words) b = -1;
bi[a] = b;
printf("\nWord: %s Position in vocabulary: %lld\n", st[a], bi[a]);
if (b == -1) {
printf("Out of dictionary word!\n");
break;
}
}
if (b == -1) continue;
printf("\n Word Cosine distance\n------------------------------------------------------------------------\n");
for (a = 0; a < size; a++) vec[a] = 0;
for (b = 0; b < cn; b++) {
if (bi[b] == -1) continue;
for (a = 0; a < size; a++) vec[a] += M[a + bi[b] * size];
}
len = 0;
for (a = 0; a < size; a++) len += vec[a] * vec[a];
len = sqrt(len);
for (a = 0; a < size; a++) vec[a] /= len;
for (a = 0; a < N; a++) bestd[a] = -1;
for (a = 0; a < N; a++) bestw[a][0] = 0;
for (c = 0; c < words; c++) {
a = 0;
for (b = 0; b < cn; b++) if (bi[b] == c) a = 1;
if (a == 1) continue;
dist = 0;
for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
for (a = 0; a < N; a++) {
if (dist > bestd[a]) {
for (d = N - 1; d > a; d--) {
bestd[d] = bestd[d - 1];
strcpy(bestw[d], bestw[d - 1]);
}
bestd[a] = dist;
strcpy(bestw[a], &vocab[c * max_w]);
break;
}
}
}
for (a = 0; a < N; a++) printf("%50s\t\t%f\n", bestw[a], bestd[a]);
}
return 0;
}
Loading

0 comments on commit f66f4cb

Please sign in to comment.