This likely isn't an issue with your code, but something to be wary of

[thistleknot]

I had some categories I had picked out manually from some lists that FRED had published and I was comparing them against what I downloaded and noticed they weren't in any of the lists that fredr downloads. So i looked into the api calls and it looks like the API stops at 1000 returns

for example

fredr_series_categories("UMCSENT") is part of 32457 but you won't see if when you do a

fredr_category_series(32457)$id

so far I've checked some others and it appears when implementing the calls they stop at 1000

I don't know how to solve this nor where to go to get all the entries per series. Another series that has a similar problem is 97 (housing)

[thistleknot]

I see there is a limit and pagination... ugh... going to have the joy of figuring out which series have more than 1000 entries

[sboysel]

The limit parameter is documented for some endpoints (e.g. fred/seriesobservations) and is not documented. My hunch is there is an implicit 1000 limit for fred/series/categories. In any case, pagination solutions are something we discuss in #68 .

[thistleknot]

Thanks. I ended up retrofitting my code to check the nrow's returned and do a do-while loop (thank you business programming lbcc) But I'm glad you are aware of it. You did add the offset paramater so I was able to make due. This can be closed. Albeit I'm sure people will want the solution.

recursive.categories <- function(x) {

  #parent <- fredr_category(32266)
  Sys.sleep(.5)
  parent <- fredr_category(x)

  Sys.sleep(.5)

  offset=0
  series <- fredr_category_series(x)
  offset=offset+1000

  repeat{
    Sys.sleep(.5)
    set2 <- fredr_category_series(x, offset=offset)

    #empty
    if(nrow(set2)==0)
    {
      break
    }
    offset=offset+1000
    series <- rbind(series,set2)

    #not empty, but no more
    if(nrow(set2)<1000){
      break
    }
  }

  Sys.sleep(.5)
  children <- fredr_category_children(category_id = parent$id)

  print(parent)
  print(series)

  if (length(children) == 0)  {
    p <- cbind(parent$id,parent$name)

    names(p) <- iconv(paste(parent$id,parent$name), from="UTF-8", to="UTF-8", sub="")
    #names(p) <- paste(parent$id,parent$name)
    #return (p)
    if(nrow(series)!=0)
    {
      #set <- list(parent,series)
      #names(set) <- c(iconv(paste(parent$id,parent$name,"info"), from="UTF-8", to="UTF-8", sub=""),iconv(paste(parent$id,parent$name,"series"), from="UTF-8", to="UTF-8", sub=""))
      set <- list(series)
      names(set) <- c(iconv(paste(parent$id,parent$name,"series"), from="UTF-8", to="UTF-8", sub=""))
    } else 
    {
      #set <- parent
    }

    return(set)
  }

  else  {
    c <- (lapply(children[["id"]], function(y)
    {
      Sys.sleep(.5)
      print(y)
      recursive.categories(y)
    }))
    names(c) <- iconv(paste(children$id,children$name), from="UTF-8", to="UTF-8", sub="")
    #names(c) <- paste(children$id,children$name)

    #set <- list(parent,c,series)
    #set <- list(c,series)

    if(nrow(series)!=0)
    {
      #set <- list(parent,c,series)
      #names(set) <- c(iconv(paste(parent$id,parent$name,"info"), from="UTF-8", to="UTF-8", sub=""),iconv(paste(parent$id,parent$name,"children"), from="UTF-8", to="UTF-8", sub=""),iconv(paste(parent$id,parent$name,"series"), from="UTF-8", to="UTF-8", sub=""))
      set <- list(c,series)
      names(set) <- c(iconv(paste(parent$id,parent$name,"children"), from="UTF-8", to="UTF-8", sub=""),iconv(paste(parent$id,parent$name,"series"), from="UTF-8", to="UTF-8", sub=""))
    } else 
    {
      #set <- list(parent,c)
      #names(set) <- c(iconv(paste(parent$id,parent$name,"info"), from="UTF-8", to="UTF-8", sub=""),iconv(paste(parent$id,parent$name,"children"), from="UTF-8", to="UTF-8", sub=""))
      set <- list(c)
      names(set) <- c(iconv(paste(parent$id,parent$name,"children"), from="UTF-8", to="UTF-8", sub=""))
    }

    return(set)
    #return(c)
  }

}

sets <- recursive.categories(0)
#sets <- recursive.categories(32266)
#sets <- recursive.categories(32998)
View(sets)

store <- as.Node(sets)
ToDataFrameTree(store)

library(data.table)
sets <- readRDS(file="sets_fred_data.RData")
setsd <- readRDS(file="fred_data.RData")

data.table::as.Node(sets)

store <- as.Node(sets)
#store$attributesAll

#View(store)

cats <- c("32455 Prices","1 Production & Business Activity","10 Population, Employment, & Labor Markets","32991 Money, Banking, & Finance","32263 International Data")

threshold <- 70

#lapply(cats, function(x)
#{#x=cats[1]
#store <- as.Node(sets$`0 Categories children`[x])
store <- as.Node(sets)
df  <- na.omit.list(store$Get(function(node) 
{
  d <- data.frame(cbind(node$id[as.numeric(node$popularity)>=threshold],node$popularity[as.numeric(node$popularity)>=threshold],node$frequency[as.numeric(node$popularity)>=threshold],node$title[as.numeric(node$popularity)>=threshold]))
  if(nrow(d)!=0)
  {
    colnames(d) <- c("id", "popularity","frequency","title")
    d
  }
  else return (NA)

}))

#secondLevel <- as.Node(df)

#secondLevel$Get("popularity", simplify="regular")
#Get(secondLevel, "id"), simplify = c("regular"))

write.csv(rbindlist(df, id=TRUE),file=paste0("0",".csv"))

#})

nrow(rbindlist(df))

length(df)