jwijffels
commented
5 years ago Project X has finished. It has wondered in embeddings space of documentation of R objects called x. The R code generating the text is put below. The text will be put into a next comment.
library(tools)
library(data.table)
library(ruimtehol)
library(udpipe)
library(digest)
library(Rfiglet)
## GET DATA
x <- installed.packages()
x <- rownames(x)
x <- lapply(x, FUN=function(pkg){
x <- Rd_db(pkg)
x <- lapply(x, FUN=function(x){
x <- tools:::.Rd_get_metadata(x, "arguments")
x <- grep('\"x\"', x, value=TRUE)
if(length(x) == 0){
return(NULL)
}
x <- lapply(x, FUN=function(x) unlist(eval(parse(text = x))))
x <- x[sapply(x, FUN=function(x) grepl("x", head(x, 1)))]
x <- sapply(x, FUN=function(x) paste(x, collapse = " "))
x <- gsub("[^[:alnum:]]", " ", x)
x <- gsub(" +", " ", x)
x
})
if(length(x) == 0){
return(NULL)
}
x <- x[sapply(x, FUN=function(x) length(x) > 0)]
x <- Map(rd = names(x), x, f=function(rd, x){
data.frame(documentationfile = rd, text = x, stringsAsFactors = FALSE)
})
x <- rbindlist(x)
x <- setDF(x)
x$rpackage <- rep(pkg, nrow(x))
x
})
x <- rbindlist(x, fill = TRUE)
x$text <- trimws(x$text)
x <- setnames(x, old = c("rpackage", "documentationfile", "text"), new = c("doc_id", "sentence_id", "text"))
x_train <- x[, list(token = tolower(unlist(strsplit(text, " ")))), by = list(doc_id, sentence_id)]
x_train <- setDF(x_train)
save(x, x_train, file = "x.RData")
load("x.RData")
## BUILD TEXT
model <- embed_sentencespace(x = x_train, early_stopping = 0.9, dim = 50,
loss = "hinge", lr = 0.10, ngrams = 3, minCount = 2,
similarity = "cosine", adagrad = TRUE, epoch = 25,
maxTrainTime = 10, negSearchLimit = 3, validationPatience = 25)
nwords <- function(x) length(unlist(strsplit(x, split = " ")))
X <- sprintf("Project x %s", "A Workbook object")
Z <- X
N <- nwords(X)
while(N < 50000){
Y <- predict(model, Z, basedoc = txt_sample(unique(x$text), n = 500), k = 50)
Z <- sample(Y$prediction$label_starspace, size = 1, prob = Y$prediction$similarity)
Z <- sprintf("Project %s", Z)
X <- sprintf("%s\n%s", X, Z)
N <- N + nwords(Z)
}
cat(X, sep = "\n", file = "X.txt")
digest(X, algo = "xxhash64")
as.numeric(starspace_embedding(model, X))
## ASCIIFY OUTPUT
set.seed(123)
X <- readLines("X.txt")
f <- file("X_figlet.txt", open = "at")
for(i in 1:length(X)){
print(sprintf("%s/%s", i, length(X)))
x <- X[i]
writeLines(sprintf("\n%s", x), con = f, sep = "\n")
x <- Rfiglet::figlet(x, font = sample(Rfiglet:::.__Rfiglet_fonts, 1))
writeLines(paste(paste(x, collapse = "\n"), sep = "\n"), con = f, sep = "\n")
}
close(f)
hugovk
Project Xwill be about showing the wonders of whatxis, what is should be, it's definitions and it's usage. Project X will wonder in documentation space of R functions. I'm an R user, allfunctions argumentsin R are documented. Many of them are are calledx. Example belowFor this submission to NaNoGenMo, I'll get all textual documentation of all arguments of all R functions which are called
xand I'll generate new possible values ofx. How will these be generated?x