glycerine
commented
8 years ago To try and reproduce this, I started over from scratch, renaming away the cockroach-data directory. Still only running one database node. This time after I kill -9 the server and bring it back up again, the client connects to the server, but the server has forgotten the database "bank" that I had created.
Severe data loss detected; the entire database is gone this time.
root@cr1:26257> set database = bank; select rowid,* from z;
root@cr1:26257> set database = bank; select rowid,* from z;
+--------------------+----+--------+-----+
| rowid | a | b | d |
+--------------------+----+--------+-----+
| 153258748090122241 | 34 | yeehaw | 123 |
| 153258822017286145 | 34 | yeehaw | 123 |
| 153258873012715521 | 34 | yeehaw | 123 |
| 153258901593817089 | 34 | yeehaw | 123 |
| 153258901597716481 | 34 | yeehaw | 123 |
+--------------------+----+--------+-----+
root@cr1:26257> ^D
< here will kill -9 and restart the server>
[jaten@cr1 ~]$ cockroach sql --insecure --host=cr1
# Welcome to the cockroach SQL interface.
# All statements must be terminated by a semicolon.
# To exit: CTRL + D.
root@cr1:26257> set database = bank; select rowid,* from z;
root@cr1:26257> set database = bank; select rowid,* from z;
pq: database "bank" does not exist
root@cr1:26257> set database = bank;
root@cr1:26257> set database = bank;
pq: database "bank" does not exist
root@cr1:26257> full transcript:
[jaten@cr1 ~]$ mv cockroach-data cockroach-data.hangs
[jaten@cr1 ~]$ cockroach start --insecure --host=cr1 --background
build: beta-20160616 @ 2016/06/16 16:51:45 (go1.6.2)
admin: http://cr1:8080
sql: postgresql://root@cr1:26257?sslmode=disable
logs: cockroach-data/logs
store[0]: path=cockroach-data
[jaten@cr1 ~]$ cockroach sql --insecure --host=cr1
# Welcome to the cockroach SQL interface.
# All statements must be terminated by a semicolon.
# To exit: CTRL + D.
root@cr1:26257> show tables;
root@cr1:26257> show tables;
pq: no database specified
root@cr1:26257> create database bank;
root@cr1:26257> create database bank;
CREATE DATABASE
root@cr1:26257> set database=bank;
root@cr1:26257> set database=bank;
SET
root@cr1:26257> show tables;
root@cr1:26257> show tables;
+-------+
| Table |
+-------+
+-------+
root@cr1:26257> create table z (rowid int, a int, b string, d bytes);
root@cr1:26257> create table z (rowid int, a int, b string, d bytes);
pq: duplicate column name: "rowid"
root@cr1:26257> create table z (a int, b string, d bytes);
root@cr1:26257> create table z (a int, b string, d bytes);
CREATE TABLE
root@cr1:26257> z
root@cr1:26257> z
-> ;
-> ;
pq: syntax error at or near "z"
z
^
root@cr1:26257> show columns from z;
root@cr1:26257> show columns from z;
+-------+--------+-------+----------------+
| Field | Type | Null | Default |
+-------+--------+-------+----------------+
| a | INT | true | NULL |
| b | STRING | true | NULL |
| d | BYTES | true | NULL |
| rowid | INT | false | unique_rowid() |
+-------+--------+-------+----------------+
root@cr1:26257> insert into z (a,b,d) values (34, 'yeehaw', b'123');
root@cr1:26257> insert into z (a,b,d) values (34, 'yeehaw', b'123');
INSERT 1
root@cr1:26257> select * from z;
root@cr1:26257> select * from z;
+----+--------+-----+
| a | b | d |
+----+--------+-----+
| 34 | yeehaw | 123 |
+----+--------+-----+
root@cr1:26257> select rowid,* from z;
root@cr1:26257> select rowid,* from z;
+--------------------+----+--------+-----+
| rowid | a | b | d |
+--------------------+----+--------+-----+
| 153258748090122241 | 34 | yeehaw | 123 |
+--------------------+----+--------+-----+
root@cr1:26257> insert into z (a,b,d) values (34, 'yeehaw', b'123');
root@cr1:26257> insert into z (a,b,d) values (34, 'yeehaw', b'123');
INSERT 1
root@cr1:26257> select rowid,* from z;
root@cr1:26257> select rowid,* from z;
+--------------------+----+--------+-----+
| rowid | a | b | d |
+--------------------+----+--------+-----+
| 153258748090122241 | 34 | yeehaw | 123 |
| 153258822017286145 | 34 | yeehaw | 123 |
+--------------------+----+--------+-----+
root@cr1:26257> insert into z (a,b,d) values (34, 'yeehaw', b'123');
root@cr1:26257> insert into z (a,b,d) values (34, 'yeehaw', b'123');
INSERT 1
root@cr1:26257> select rowid,* from z;
root@cr1:26257> select rowid,* from z;
+--------------------+----+--------+-----+
| rowid | a | b | d |
+--------------------+----+--------+-----+
| 153258748090122241 | 34 | yeehaw | 123 |
| 153258822017286145 | 34 | yeehaw | 123 |
| 153258873012715521 | 34 | yeehaw | 123 |
+--------------------+----+--------+-----+
root@cr1:26257> insert into z (a,b,d) values (34, 'yeehaw', b'123');insert into z (a,b,d) values (34, 'yeehaw', b'123');
root@cr1:26257> insert into z (a,b,d) values (34, 'yeehaw', b'123');insert into z (a,b,d) values (34, 'yeehaw', b'123');
INSERT 1
root@cr1:26257> select rowid,* from z;
root@cr1:26257> select rowid,* from z;
+--------------------+----+--------+-----+
| rowid | a | b | d |
+--------------------+----+--------+-----+
| 153258748090122241 | 34 | yeehaw | 123 |
| 153258822017286145 | 34 | yeehaw | 123 |
| 153258873012715521 | 34 | yeehaw | 123 |
| 153258901593817089 | 34 | yeehaw | 123 |
| 153258901597716481 | 34 | yeehaw | 123 |
+--------------------+----+--------+-----+
root@cr1:26257> ^D
[jaten@cr1 ~]$ cockroach sql --insecure --host=cr1
# Welcome to the cockroach SQL interface.
# All statements must be terminated by a semicolon.
# To exit: CTRL + D.
root@cr1:26257> set database = bank; select rowid,* from z;
root@cr1:26257> set database = bank; select rowid,* from z;
+--------------------+----+--------+-----+
| rowid | a | b | d |
+--------------------+----+--------+-----+
| 153258748090122241 | 34 | yeehaw | 123 |
| 153258822017286145 | 34 | yeehaw | 123 |
| 153258873012715521 | 34 | yeehaw | 123 |
| 153258901593817089 | 34 | yeehaw | 123 |
| 153258901597716481 | 34 | yeehaw | 123 |
+--------------------+----+--------+-----+
root@cr1:26257> ^D
[jaten@cr1 ~]$ cockroach sql --insecure --host=cr1
# Welcome to the cockroach SQL interface.
# All statements must be terminated by a semicolon.
# To exit: CTRL + D.
root@cr1:26257> set database = bank; select rowid,* from z;
root@cr1:26257> set database = bank; select rowid,* from z;
pq: database "bank" does not exist
root@cr1:26257> set database = bank;
root@cr1:26257> set database = bank;
pq: database "bank" does not exist
root@cr1:26257> ^D
Error: pq: database "bank" does not exist
Failed running "sql"
[jaten@cr1 ~]$ ls
cockroach-data cockroach-data.hangs cockroach-data.tar.gz cr.hang.txt go pkg
[jaten@cr1 ~]$ 
mberhault
On linux/amd64 Centos7.2 [digital ocean, 512MB droplet] with cockroach version: that fetched with wget https://binaries.cockroachdb.com/cockroach-beta-20160616.linux-amd64.tgz
When log files are not available, supply the output of
cockroach versionand all flags/environment variables passed tocockroach startinstead.this is how cockroach was started:
a) I started the server: $ cockroach start --insecure --host=cr1
b) I killed the cockroach server with "kill -9 pid". Then I brought it back up again with $ cockroach start --insecure --host=cr1 --background
c) then I tried to connect to it with $ cockroach sql which didn't work at all, but then I tried $ cockroach sql --host=cr1 which seemed to work but then hung forever when I issued "show databases;"
What I expected: no hang on trying to do "show databases" What I observed: hang forever, waiting for some response from the server.
Attaching log files. Stack traces of server and client I'll paste below. cockroach-data.tar.gz
stack traces: