mongod start failed (monogdb 3.0.2, database storageEngine-wiredTiger)

Question

I was using mongodb for a month yesterday morning after unusual shutdown my mongodb is failing to start. Here I am using wiredTiger Storage Engine, I think the wiredTiger.wt file got corrupted but I dont know what to do

on doing sevice mongod start it gave me this error log

 2015-05-12T16:54:35.442+0530 I CONTROL ***** SERVER RESTARTED *****
2015-05-12T16:54:35.465+0530 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=7G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
2015-05-12T16:54:35.469+0530 E STORAGE [initandlisten] WiredTiger (0) [1431429875:469521][10501:0x7f1a672e3c20], file:WiredTiger.wt, connection: read checksum error [4096B @ 32768, 3884066602 != 1841814112]
2015-05-12T16:54:35.469+0530 E STORAGE [initandlisten] WiredTiger (0) [1431429875:469550][10501:0x7f1a672e3c20], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value
2015-05-12T16:54:35.469+0530 E STORAGE [initandlisten] WiredTiger (-31804) [1431429875:469563][10501:0x7f1a672e3c20], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic
2015-05-12T16:54:35.469+0530 I - [initandlisten] Fatal Assertion 28558
2015-05-12T16:54:35.478+0530 I CONTROL [initandlisten] 
0xf68719 0xf06c91 0xeeaad1 0xd8761a 0x1399239 0x13993f5 0x1399894 0x12f0e52 0x130955c 0x1307358 0x13081d2 0x133014b 0x139877d 0x13669eb 0x132d937 0xd7150b 0xd6f3a8 0xa8033d 0x7f36a2 0x7f8c14 0x3bf4a1ed5d 0x7f1435
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B68719"},{"b":"400000","o":"B06C91"},{"b":"400000","o":"AEAAD1"},{"b":"400000","o":"98761A"},{"b":"400000","o":"F99239"},{"b":"400000","o":"F993F5"},{"b":"400000","o":"F99894"},{"b":"400000","o":"EF0E52"},{"b":"400000","o":"F0955C"},{"b":"400000","o":"F07358"},{"b":"400000","o":"F081D2"},{"b":"400000","o":"F3014B"},{"b":"400000","o":"F9877D"},{"b":"400000","o":"F669EB"},{"b":"400000","o":"F2D937"},{"b":"400000","o":"97150B"},{"b":"400000","o":"96F3A8"},{"b":"400000","o":"68033D"},{"b":"400000","o":"3F36A2"},{"b":"400000","o":"3F8C14"},{"b":"3BF4A00000","o":"1ED5D"},{"b":"400000","o":"3F1435"}],"processInfo":{ "mongodbVersion" : "3.0.2", "gitVersion" : "6201872043ecbbc0a4cc169b5482dcf385fc464f", "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.12.2.el6.x86_64", "version" : "#1 SMP Wed Mar 11 22:03:14 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "528324F7E7422ECA899108CA2A12B5AAAA2A5569" }, { "b" : "7FFFB0EFE000", "elfType" : 3, "buildId" : "4B8E1260CCD7C3D8CC131E84001F7220651617EC" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "934508308DAF0D5C61E9997463F0D8B0A3F096BA" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "A4329A30669C783FA8DEEB7D1EA83749A8FA14E1" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "F07F2E7CF4BFB393CC9BBE8CDC6463652E14DB07" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7D8E9374F4A4EA38A7C1E763F32240EA113E4208" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "246C3BAB0AB093AFD59D34C8CBF29E786DE4BE97" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "E4EAB3C200B7D8444FF95AB01F6466924A6A5F5F" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F8E59B70E469F3A924A268911FF8FD0C37E7460" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "54BA6B78A9220344E77463947215E42F0EABCC62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6797403AA5F8FAD8ADFF683478B45F528CE4FB0E" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "8CE28F280150E62296240E70ECAC64E4A57AB826" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "05733977F4E41652B86070B27A0CFC2C1EA7719D" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E3FA235F3BA3F776A01A18ECA737C9890F445923" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F8B68F301C19BF06AF56B4B06E0A69F89D2C1F8D" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf68719]
mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf06c91]
mongod(_ZN5mongo13fassertFailedEi+0x61) [0xeeaad1]
mongod(+0x98761A) [0xd8761a]
mongod(__wt_eventv+0x489) [0x1399239]
mongod(__wt_err+0x95) [0x13993f5]
mongod(__wt_panic+0x24) [0x1399894]
mongod(__wt_bm_read+0x72) [0x12f0e52]
mongod(__wt_bt_read+0x1AC) [0x130955c]
mongod(__wt_btree_tree_open+0x58) [0x1307358]
mongod(__wt_btree_open+0xD02) [0x13081d2]
mongod(__wt_conn_btree_get+0x19B) [0x133014b]
mongod(__wt_session_get_btree+0x31D) [0x139877d]
mongod(__wt_metadata_open+0x2B) [0x13669eb]
mongod(wiredtiger_open+0xCD7) [0x132d937]
mongod(_ZN5mongo18WiredTigerKVEngineC1ERKSsS2_bb+0x2EB) [0xd7150b]
mongod(+0x96F3A8) [0xd6f3a8]
mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa8033d]
mongod(_ZN5mongo13initAndListenEi+0x422) [0x7f36a2]
mongod(main+0x134) [0x7f8c14]
libc.so.6(__libc_start_main+0xFD) [0x3bf4a1ed5d]
mongod(+0x3F1435) [0x7f1435]
----- END BACKTRACE -----
2015-05-12T16:54:35.478+0530 I - [initandlisten]

***aborting after fassert() failure

and even unable to repair it, giving similar type of error when I try to repair it.

[root@host name wiredTiger]# mongod --dbpath /home/mongodbProject/database/wiredTiger --repair --repairpath /home/mongodbProject/repairData/rep --storageEngine wiredTiger
2015-05-13T16:30:15.206+0530 I STORAGE  [initandlisten] wiredtiger_open config: create,cache_size=7G,session_max=20000,eviction=(threads_max=4),statistics=(fast),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
2015-05-13T16:30:15.546+0530 I STORAGE  [initandlisten] Repairing size cache
2015-05-13T16:30:15.554+0530 E STORAGE  [initandlisten] WiredTiger (0) [1431514815:554248][5946:0x7fcb2c444c20], file:sizeStorer.wt, session.verify: read checksum error [4096B @ 40960, 2019257843 != 0]
2015-05-13T16:30:15.554+0530 E STORAGE  [initandlisten] WiredTiger (0) [1431514815:554309][5946:0x7fcb2c444c20], file:sizeStorer.wt, session.verify: sizeStorer.wt: encountered an illegal file format or internal value
2015-05-13T16:30:15.554+0530 E STORAGE  [initandlisten] WiredTiger (-31804) [1431514815:554334][5946:0x7fcb2c444c20], file:sizeStorer.wt, session.verify: the process must exit and restart: WT_PANIC: WiredTiger library panic
2015-05-13T16:30:15.554+0530 I -        [initandlisten] Fatal Assertion 28558
2015-05-13T16:30:15.571+0530 I CONTROL  [initandlisten] 
 0xf68719 0xf06c91 0xeeaad1 0xd8761a 0x1399239 0x13993f5 0x1399894 0x12eeb3e 0x12eefd8 0x12f1bff 0x12f1d78 0x131b32a 0x139489e 0x1394ad8 0x1394f66 0xd6fc85 0xd71846 0xd6f3a8 0xa8033d 0x7f36a2 0x7f8c14 0x3bf4a1ed5d 0x7f1435
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B68719"},{"b":"400000","o":"B06C91"},{"b":"400000","o":"AEAAD1"},{"b":"400000","o":"98761A"},{"b":"400000","o":"F99239"},{"b":"400000","o":"F993F5"},{"b":"400000","o":"F99894"},{"b":"400000","o":"EEEB3E"},{"b":"400000","o":"EEEFD8"},{"b":"400000","o":"EF1BFF"},{"b":"400000","o":"EF1D78"},{"b":"400000","o":"F1B32A"},{"b":"400000","o":"F9489E"},{"b":"400000","o":"F94AD8"},{"b":"400000","o":"F94F66"},{"b":"400000","o":"96FC85"},{"b":"400000","o":"971846"},{"b":"400000","o":"96F3A8"},{"b":"400000","o":"68033D"},{"b":"400000","o":"3F36A2"},{"b":"400000","o":"3F8C14"},{"b":"3BF4A00000","o":"1ED5D"},{"b":"400000","o":"3F1435"}],"processInfo":{ "mongodbVersion" : "3.0.2", "gitVersion" : "6201872043ecbbc0a4cc169b5482dcf385fc464f", "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.12.2.el6.x86_64", "version" : "#1 SMP Wed Mar 11 22:03:14 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "528324F7E7422ECA899108CA2A12B5AAAA2A5569" }, { "b" : "7FFF982F7000", "elfType" : 3, "buildId" : "4B8E1260CCD7C3D8CC131E84001F7220651617EC" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "934508308DAF0D5C61E9997463F0D8B0A3F096BA" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "A4329A30669C783FA8DEEB7D1EA83749A8FA14E1" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "F07F2E7CF4BFB393CC9BBE8CDC6463652E14DB07" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7D8E9374F4A4EA38A7C1E763F32240EA113E4208" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "246C3BAB0AB093AFD59D34C8CBF29E786DE4BE97" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "E4EAB3C200B7D8444FF95AB01F6466924A6A5F5F" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F8E59B70E469F3A924A268911FF8FD0C37E7460" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "54BA6B78A9220344E77463947215E42F0EABCC62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6797403AA5F8FAD8ADFF683478B45F528CE4FB0E" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "8CE28F280150E62296240E70ECAC64E4A57AB826" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "05733977F4E41652B86070B27A0CFC2C1EA7719D" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E3FA235F3BA3F776A01A18ECA737C9890F445923" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F8B68F301C19BF06AF56B4B06E0A69F89D2C1F8D" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf68719]
 mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf06c91]
 mongod(_ZN5mongo13fassertFailedEi+0x61) [0xeeaad1]
 mongod(+0x98761A) [0xd8761a]
 mongod(__wt_eventv+0x489) [0x1399239]
 mongod(__wt_err+0x95) [0x13993f5]
 mongod(__wt_panic+0x24) [0x1399894]
 mongod(__wt_block_extlist_read+0x6E) [0x12eeb3e]
 mongod(__wt_block_extlist_read_avail+0x28) [0x12eefd8]
 mongod(+0xEF1BFF) [0x12f1bff]
 mongod(__wt_block_verify_start+0x108) [0x12f1d78]
 mongod(__wt_verify+0x4AA) [0x131b32a]
 mongod(__wt_schema_worker+0x35E) [0x139489e]
 mongod(__wt_schema_worker+0x598) [0x1394ad8]
 mongod(+0xF94F66) [0x1394f66]
 mongod(_ZN5mongo18WiredTigerKVEngine16_salvageIfNeededEPKc+0x45) [0xd6fc85]
 mongod(_ZN5mongo18WiredTigerKVEngineC1ERKSsS2_bb+0x626) [0xd71846]
 mongod(+0x96F3A8) [0xd6f3a8]
 mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa8033d]
 mongod(_ZN5mongo13initAndListenEi+0x422) [0x7f36a2]
 mongod(main+0x134) [0x7f8c14]
 libc.so.6(__libc_start_main+0xFD) [0x3bf4a1ed5d]
 mongod(+0x3F1435) [0x7f1435]
-----  END BACKTRACE  -----
2015-05-13T16:30:15.571+0530 I -        [initandlisten] 

***aborting after fassert() failure

I have tried searching for similar and tried most of it but dint work in my case.Please help

Any help would be appreciated

Solution

Issue solved : https://jira.mongodb.org/browse/SERVER-18448

there were some files corrupted after updating those files "sizeStorer.wt, WiredTiger.turtle, WiredTiger.wt" able to repair database using

sudo mongod --repair --dbpath /database/db --storageEngine wiredTiger

before restart check permissions of all files present in database and after updating their permissions restart server.

sudo service mongod restart

there may be some issues regarding lock files. So, delete mongod.lock, wiredTiger.lock and mongodb.sock file present in /tmp folder.

now, restart server.

Comments

[BSQ]

You made my day ! Thank you

[asa]

that's doesn't work for me. As soonas I run the first command you mentioned, i get the same error.