search

JetBrains-Research / pubtrends

Scientific literature explorer. Runs a Pubmed or Semantic Scholar search and allows user to explore high-level structure of result papers
Apache License 2.0
35 stars 2 forks source link

Problem updating new papers #320

Open olegs opened 1 year ago

olegs commented 1 year ago

Update Pubmed publications using commandline:

java -cp pubtrends-0.21.1300.jar org.jetbrains.bio.pubtrends.pm.PubmedLoader --fillDatabase | tee -a crontab_update.log

Update material view fails.

2022-10-14 08:03:04,847 INFO [main] o.j.b.p.d.PubmedPostgresWriter [PubmedPostgresWriter.kt:188] Update material view matview_pmcitations
2022-10-14 08:03:53,434 WARN [main] Exposed [ThreadLocalTransactionManager.kt:169] Transaction attempt #0 failed: org.postgresql.util.PSQLException: An I/O error occurred while sending to the backend.. Statement(s): 
                    do
                    $$
                    begin
                    IF exists (select matviewname from pg_matviews where matviewname = 'matview_pmcitations') THEN
                        refresh materialized view matview_pmcitations;
                    END IF;
                    end;
                    $$;

org.jetbrains.exposed.exceptions.ExposedSQLException: org.postgresql.util.PSQLException: An I/O error occurred while sending to the backend.
    at org.jetbrains.exposed.sql.statements.Statement.executeIn$exposed(Statement.kt:61)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:184)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:126)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:102)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:93)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter$close$1.invoke(PubmedPostgresWriter.kt:189)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter$close$1.invoke(PubmedPostgresWriter.kt:187)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.inTopLevelTransaction$run(ThreadLocalTransactionManager.kt:156)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.access$inTopLevelTransaction$run(ThreadLocalTransactionManager.kt:1)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt$inTopLevelTransaction$1.invoke(ThreadLocalTransactionManager.kt:197)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.keepAndRestoreTransactionRefAfterRun(ThreadLocalTransactionManager.kt:205)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.inTopLevelTransaction(ThreadLocalTransactionManager.kt:196)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt$transaction$1.invoke(ThreadLocalTransactionManager.kt:134)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.keepAndRestoreTransactionRefAfterRun(ThreadLocalTransactionManager.kt:205)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction(ThreadLocalTransactionManager.kt:106)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction(ThreadLocalTransactionManager.kt:104)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction$default(ThreadLocalTransactionManager.kt:103)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter.close(PubmedPostgresWriter.kt:187)
    at kotlin.io.CloseableKt.closeFinally(Closeable.kt:57)
    at org.jetbrains.bio.pubtrends.pm.PubmedLoader.main(PubmedLoader.kt:65)
Caused by: org.postgresql.util.PSQLException: An I/O error occurred while sending to the backend.
    at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:383)
    at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:490)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:408)
    at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:181)
    at org.postgresql.jdbc.PgPreparedStatement.executeUpdate(PgPreparedStatement.java:149)
    at org.jetbrains.exposed.sql.Transaction$exec$2.executeInternal(Transaction.kt:107)
    at org.jetbrains.exposed.sql.statements.Statement.executeIn$exposed(Statement.kt:59)
    ... 19 common frames omitted
Caused by: java.io.EOFException: null
    at org.postgresql.core.PGStream.receiveChar(PGStream.java:455)
    at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2120)
    at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:356)
    ... 25 common frames omitted
2022-10-14 08:03:53,437 INFO [main] o.j.b.p.d.PubmedPostgresWriter [PubmedPostgresWriter.kt:188] Update material view matview_pmcitations
2022-10-14 08:03:53,453 WARN [main] Exposed [ThreadLocalTransactionManager.kt:169] Transaction attempt #1 failed: org.postgresql.util.PSQLException: FATAL: the database system is in recovery mode. Statement(s): 
                    do
                    $$
                    begin
                    IF exists (select matviewname from pg_matviews where matviewname = 'matview_pmcitations') THEN
                        refresh materialized view matview_pmcitations;
                    END IF;
                    end;
                    $$;

org.jetbrains.exposed.exceptions.ExposedSQLException: org.postgresql.util.PSQLException: FATAL: the database system is in recovery mode
    at org.jetbrains.exposed.sql.statements.Statement.executeIn$exposed(Statement.kt:48)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:184)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:126)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:102)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:93)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter$close$1.invoke(PubmedPostgresWriter.kt:189)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter$close$1.invoke(PubmedPostgresWriter.kt:187)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.inTopLevelTransaction$run(ThreadLocalTransactionManager.kt:156)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.access$inTopLevelTransaction$run(ThreadLocalTransactionManager.kt:1)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt$inTopLevelTransaction$1.invoke(ThreadLocalTransactionManager.kt:197)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.keepAndRestoreTransactionRefAfterRun(ThreadLocalTransactionManager.kt:205)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.inTopLevelTransaction(ThreadLocalTransactionManager.kt:196)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt$transaction$1.invoke(ThreadLocalTransactionManager.kt:134)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.keepAndRestoreTransactionRefAfterRun(ThreadLocalTransactionManager.kt:205)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction(ThreadLocalTransactionManager.kt:106)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction(ThreadLocalTransactionManager.kt:104)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction$default(ThreadLocalTransactionManager.kt:103)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter.close(PubmedPostgresWriter.kt:187)
    at kotlin.io.CloseableKt.closeFinally(Closeable.kt:57)
    at org.jetbrains.bio.pubtrends.pm.PubmedLoader.main(PubmedLoader.kt:65)
Caused by: org.postgresql.util.PSQLException: FATAL: the database system is in recovery mode
    at org.postgresql.core.v3.ConnectionFactoryImpl.doAuthentication(ConnectionFactoryImpl.java:646)
    at org.postgresql.core.v3.ConnectionFactoryImpl.tryConnect(ConnectionFactoryImpl.java:180)
    at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:235)
    at org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:49)
    at org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:223)
    at org.postgresql.Driver.makeConnection(Driver.java:402)
    at org.postgresql.Driver.connect(Driver.java:261)
    at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:677)
    at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:228)
    at org.jetbrains.exposed.sql.Database$Companion$connect$7.invoke(Database.kt:196)
    at org.jetbrains.exposed.sql.Database$Companion$connect$7.invoke(Database.kt:196)
    at org.jetbrains.exposed.sql.Database$Companion$doConnect$3.invoke(Database.kt:175)
    at org.jetbrains.exposed.sql.Database$Companion$doConnect$3.invoke(Database.kt:176)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager$ThreadLocalTransaction$connectionLazy$1.invoke(ThreadLocalTransactionManager.kt:40)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager$ThreadLocalTransaction$connectionLazy$1.invoke(ThreadLocalTransactionManager.kt:39)
    at kotlin.UnsafeLazyImpl.getValue(Lazy.kt:81)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager$ThreadLocalTransaction.getConnection(ThreadLocalTransactionManager.kt:46)
    at org.jetbrains.exposed.sql.Transaction.getConnection(Transaction.kt)
    at org.jetbrains.exposed.sql.statements.Statement.prepared(Statement.kt:25)
    at org.jetbrains.exposed.sql.statements.Statement.executeIn$exposed(Statement.kt:46)
    ... 19 common frames omitted
2022-10-14 08:03:53,454 INFO [main] o.j.b.p.d.PubmedPostgresWriter [PubmedPostgresWriter.kt:188] Update material view matview_pmcitations
2022-10-14 08:03:53,460 WARN [main] Exposed [ThreadLocalTransactionManager.kt:169] Transaction attempt #2 failed: org.postgresql.util.PSQLException: FATAL: the database system is in recovery mode. Statement(s): 
                    do
                    $$
                    begin
                    IF exists (select matviewname from pg_matviews where matviewname = 'matview_pmcitations') THEN
                        refresh materialized view matview_pmcitations;
                    END IF;
                    end;
                    $$;

org.jetbrains.exposed.exceptions.ExposedSQLException: org.postgresql.util.PSQLException: FATAL: the database system is in recovery mode
    at org.jetbrains.exposed.sql.statements.Statement.executeIn$exposed(Statement.kt:48)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:184)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:126)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:102)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:93)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter$close$1.invoke(PubmedPostgresWriter.kt:189)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter$close$1.invoke(PubmedPostgresWriter.kt:187)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.inTopLevelTransaction$run(ThreadLocalTransactionManager.kt:156)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.access$inTopLevelTransaction$run(ThreadLocalTransactionManager.kt:1)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt$inTopLevelTransaction$1.invoke(ThreadLocalTransactionManager.kt:197)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.keepAndRestoreTransactionRefAfterRun(ThreadLocalTransactionManager.kt:205)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.inTopLevelTransaction(ThreadLocalTransactionManager.kt:196)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt$transaction$1.invoke(ThreadLocalTransactionManager.kt:134)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.keepAndRestoreTransactionRefAfterRun(ThreadLocalTransactionManager.kt:205)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction(ThreadLocalTransactionManager.kt:106)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction(ThreadLocalTransactionManager.kt:104)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction$default(ThreadLocalTransactionManager.kt:103)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter.close(PubmedPostgresWriter.kt:187)
    at kotlin.io.CloseableKt.closeFinally(Closeable.kt:57)
    at org.jetbrains.bio.pubtrends.pm.PubmedLoader.main(PubmedLoader.kt:65)
Caused by: org.postgresql.util.PSQLException: FATAL: the database system is in recovery mode
    at org.postgresql.core.v3.ConnectionFactoryImpl.doAuthentication(ConnectionFactoryImpl.java:646)
    at org.postgresql.core.v3.ConnectionFactoryImpl.tryConnect(ConnectionFactoryImpl.java:180)
    at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:235)
    at org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:49)
    at org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:223)
    at org.postgresql.Driver.makeConnection(Driver.java:402)
    at org.postgresql.Driver.connect(Driver.java:261)
    at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:677)
    at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:228)
    at org.jetbrains.exposed.sql.Database$Companion$connect$7.invoke(Database.kt:196)
    at org.jetbrains.exposed.sql.Database$Companion$connect$7.invoke(Database.kt:196)
    at org.jetbrains.exposed.sql.Database$Companion$doConnect$3.invoke(Database.kt:175)
    at org.jetbrains.exposed.sql.Database$Companion$doConnect$3.invoke(Database.kt:176)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager$ThreadLocalTransaction$connectionLazy$1.invoke(ThreadLocalTransactionManager.kt:40)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager$ThreadLocalTransaction$connectionLazy$1.invoke(ThreadLocalTransactionManager.kt:39)
    at kotlin.UnsafeLazyImpl.getValue(Lazy.kt:81)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager$ThreadLocalTransaction.getConnection(ThreadLocalTransactionManager.kt:46)
    at org.jetbrains.exposed.sql.Transaction.getConnection(Transaction.kt)
    at org.jetbrains.exposed.sql.statements.Statement.prepared(Statement.kt:25)
    at org.jetbrains.exposed.sql.statements.Statement.executeIn$exposed(Statement.kt:46)
    ... 19 common frames omitted
Exception in thread "main" org.jetbrains.exposed.exceptions.ExposedSQLException: org.postgresql.util.PSQLException: FATAL: the database system is in recovery mode
SQL: [
                    do
                    $$
                    begin
                    IF exists (select matviewname from pg_matviews where matviewname = 'matview_pmcitations') THEN
                        refresh materialized view matview_pmcitations;
                    END IF;
                    end;
                    $$;
                    ]
    at org.jetbrains.exposed.sql.statements.Statement.executeIn$exposed(Statement.kt:48)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:184)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:126)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:102)
    at org.jetbrains.exposed.sql.Transaction.exec(Transaction.kt:93)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter$close$1.invoke(PubmedPostgresWriter.kt:189)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter$close$1.invoke(PubmedPostgresWriter.kt:187)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.inTopLevelTransaction$run(ThreadLocalTransactionManager.kt:156)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.access$inTopLevelTransaction$run(ThreadLocalTransactionManager.kt:1)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt$inTopLevelTransaction$1.invoke(ThreadLocalTransactionManager.kt:197)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.keepAndRestoreTransactionRefAfterRun(ThreadLocalTransactionManager.kt:205)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.inTopLevelTransaction(ThreadLocalTransactionManager.kt:196)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt$transaction$1.invoke(ThreadLocalTransactionManager.kt:134)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.keepAndRestoreTransactionRefAfterRun(ThreadLocalTransactionManager.kt:205)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction(ThreadLocalTransactionManager.kt:106)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction(ThreadLocalTransactionManager.kt:104)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManagerKt.transaction$default(ThreadLocalTransactionManager.kt:103)
    at org.jetbrains.bio.pubtrends.db.PubmedPostgresWriter.close(PubmedPostgresWriter.kt:187)
    at kotlin.io.CloseableKt.closeFinally(Closeable.kt:57)
    at org.jetbrains.bio.pubtrends.pm.PubmedLoader.main(PubmedLoader.kt:65)
Caused by: org.postgresql.util.PSQLException: FATAL: the database system is in recovery mode
    at org.postgresql.core.v3.ConnectionFactoryImpl.doAuthentication(ConnectionFactoryImpl.java:646)
    at org.postgresql.core.v3.ConnectionFactoryImpl.tryConnect(ConnectionFactoryImpl.java:180)
    at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:235)
    at org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:49)
    at org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:223)
    at org.postgresql.Driver.makeConnection(Driver.java:402)
    at org.postgresql.Driver.connect(Driver.java:261)
    at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:677)
    at java.sql/java.sql.DriverManager.getConnection(DriverManager.java:228)
    at org.jetbrains.exposed.sql.Database$Companion$connect$7.invoke(Database.kt:196)
    at org.jetbrains.exposed.sql.Database$Companion$connect$7.invoke(Database.kt:196)
    at org.jetbrains.exposed.sql.Database$Companion$doConnect$3.invoke(Database.kt:175)
    at org.jetbrains.exposed.sql.Database$Companion$doConnect$3.invoke(Database.kt:176)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager$ThreadLocalTransaction$connectionLazy$1.invoke(ThreadLocalTransactionManager.kt:40)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager$ThreadLocalTransaction$connectionLazy$1.invoke(ThreadLocalTransactionManager.kt:39)
    at kotlin.UnsafeLazyImpl.getValue(Lazy.kt:81)
    at org.jetbrains.exposed.sql.transactions.ThreadLocalTransactionManager$ThreadLocalTransaction.getConnection(ThreadLocalTransactionManager.kt:46)
    at org.jetbrains.exposed.sql.Transaction.getConnection(Transaction.kt)
    at org.jetbrains.exposed.sql.statements.Statement.prepared(Statement.kt:25)
    at org.jetbrains.exposed.sql.statements.Statement.executeIn$exposed(Statement.kt:46)
    ... 19 more
olegs commented 1 year ago

Was able to reproduce it from Postgres client, on the second time everything went smoothly.