user1955162's questions

I have a mount point in Linux that does not show any open files in lsof or fuser. The mount point was exported in NFS, but is no longer exported, and there are no more files or directories in the mount point. We have checked to make sure no one is in the directory, going so far as to count and check the bash processes as well. At this point, we are absolutely stumped as to why we are unable to unmount the mount points.

[root@head-02 proc]# umount /PoolB/daily.2020.12.13
umount: /PoolB/daily.2020.12.13: target is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))

We do not have the directory exported in NFS:

[root@head-02 proc]# showmount -e localhost
Export list for localhost:
/PoolB/LegacyExchange 172.18.10.0/24
/PoolB/Customers      172.18.10.0/24
/PoolB/Import         172.18.10.0/24

Output from FUSER:

[root@head-02 proc]# fuser -vm /PoolB/daily.2020.12.13
                 USER        PID ACCESS COMMAND
/PoolB/daily.2020.12.13:
                 root     kernel mount /PoolB/daily.2020.12.13

Output from LSOF:

[root@head-02 proc]# lsof +D /PoolB/daily.2020.12.13
[root@head-02 proc]# 

The system is a ZFS On Linux system, so I could try to change the mount point there or delete it. The ultimate goal is to delete the filesystem from the ZPOOL:

[root@head-02 proc]# zfs set mountpoint=legacy PoolB/daily.2020.12.13
umount: /PoolB/daily.2020.12.13: target is busy.
        (In some cases useful info about processes that use
        the device is found by lsof(8) or fuser(1))
cannot unmount '/PoolB/daily.2020.12.13': umount failed
[root@head-02 proc]# 

I even tried just deleting the filesystem:

[root@head-02 proc]# zfs destroy PoolB/daily.2020.12.13
umount: /PoolB/daily.2020.12.13: target is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
cannot unmount '/PoolB/daily.2020.12.13': umount failed
[root@head-02 proc]# 

Out of desperation, here is the strace output of the umount command that fails:

[root@head-02 proc]# strace umount /PoolB/daily.2020.12.13
execve("/usr/bin/umount", ["umount", "/PoolB/daily.2020.12.13"], 0x7ffcd33fef18 /* 22 vars */) = 0
brk(NULL)                               = 0x55d4721b9000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4088778000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=22424, ...}) = 0
mmap(NULL, 22424, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f4088772000
close(3)                                = 0
open("/lib64/libmount.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\237\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=277808, ...}) = 0
mmap(NULL, 2370944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4088315000
mprotect(0x7f4088355000, 2097152, PROT_NONE) = 0
mmap(0x7f4088555000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x40000) = 0x7f4088555000
mmap(0x7f4088557000, 3456, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f4088557000
close(3)                                = 0
open("/lib64/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\207\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=265600, ...}) = 0
mmap(NULL, 2358152, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f40880d5000
mprotect(0x7f4088111000, 2093056, PROT_NONE) = 0
mmap(0x7f4088310000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3b000) = 0x7f4088310000
mmap(0x7f4088314000, 2952, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f4088314000
close(3)                                = 0
open("/lib64/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\24\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=20064, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4088771000
mmap(NULL, 2113920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4087ed0000
mprotect(0x7f4087ed4000, 2093056, PROT_NONE) = 0
mmap(0x7f40880d3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f40880d3000
close(3)                                = 0
open("/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320i\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=155784, ...}) = 0
mmap(NULL, 2255184, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4087ca9000
mprotect(0x7f4087ccd000, 2093056, PROT_NONE) = 0
mmap(0x7f4087ecc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7f4087ecc000
mmap(0x7f4087ece000, 6480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f4087ece000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20&\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2156160, ...}) = 0
mmap(NULL, 3985888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f40878db000
mprotect(0x7f4087a9e000, 2097152, PROT_NONE) = 0
mmap(0x7f4087c9e000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c3000) = 0x7f4087c9e000
mmap(0x7f4087ca4000, 16864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f4087ca4000
close(3)                                = 0
open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\25\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=402384, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4088770000
mmap(NULL, 2494984, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4087679000
mprotect(0x7f40876d9000, 2097152, PROT_NONE) = 0
mmap(0x7f40878d9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x60000) = 0x7f40878d9000
close(3)                                = 0
open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19288, ...}) = 0
mmap(NULL, 2109712, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4087475000
mprotect(0x7f4087477000, 2097152, PROT_NONE) = 0
mmap(0x7f4087677000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f4087677000
close(3)                                = 0
open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@m\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=142232, ...}) = 0
mmap(NULL, 2208904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4087259000
mprotect(0x7f4087270000, 2093056, PROT_NONE) = 0
mmap(0x7f408746f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f408746f000
mmap(0x7f4087471000, 13448, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f4087471000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f408876f000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f408876e000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f408876c000
arch_prctl(ARCH_SET_FS, 0x7f408876c880) = 0
mprotect(0x7f4087c9e000, 16384, PROT_READ) = 0
mprotect(0x7f408746f000, 4096, PROT_READ) = 0
mprotect(0x7f4087677000, 4096, PROT_READ) = 0
mprotect(0x7f40878d9000, 4096, PROT_READ) = 0
mprotect(0x7f4087ecc000, 4096, PROT_READ) = 0
mprotect(0x7f40880d3000, 4096, PROT_READ) = 0
mprotect(0x7f4088310000, 12288, PROT_READ) = 0
mprotect(0x7f4088555000, 4096, PROT_READ) = 0
mprotect(0x55d470fcf000, 4096, PROT_READ) = 0
mprotect(0x7f4088779000, 4096, PROT_READ) = 0
munmap(0x7f4088772000, 22424)           = 0
set_tid_address(0x7f408876cb50)         = 37241
set_robust_list(0x7f408876cb60, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7f408725f820, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f40872685f0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7f408725f8b0, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f40872685f0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7fffc7932c50) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7fffc7932c50)      = -1 ENOENT (No such file or directory)
brk(NULL)                               = 0x55d4721b9000
brk(0x55d4721da000)                     = 0x55d4721da000
open("/proc/filesystems", O_RDONLY)     = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4088777000
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tr"..., 1024) = 350
stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7fffc7932830) = -1 ENOENT (No such file or directory)
read(3, "", 1024)                       = 0
close(3)                                = 0
munmap(0x7f4088777000, 4096)            = 0
access("/etc/selinux/config", F_OK)     = 0
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=106075056, ...}) = 0
mmap(NULL, 106075056, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f4080d2f000
close(3)                                = 0
getuid()                                = 0
geteuid()                               = 0
getuid()                                = 0
geteuid()                               = 0
getgid()                                = 0
getegid()                               = 0
prctl(PR_GET_DUMPABLE)                  = 1 (SUID_DUMP_USER)
lstat("/etc/mtab", {st_mode=S_IFLNK|0777, st_size=17, ...}) = 0
getuid()                                = 0
geteuid()                               = 0
getgid()                                = 0
getegid()                               = 0
prctl(PR_GET_DUMPABLE)                  = 1 (SUID_DUMP_USER)
stat("/run", {st_mode=S_IFDIR|0755, st_size=960, ...}) = 0
lstat("/run/mount/utab", 0x7fffc7932b50) = -1 ENOENT (No such file or directory)
mkdir("/run/mount", 0755)               = -1 EEXIST (File exists)
stat("/run/mount/utab", 0x7fffc7932a40) = -1 ENOENT (No such file or directory)
stat("/run/mount", {st_mode=S_IFDIR|0755, st_size=40, ...}) = 0
geteuid()                               = 0
getegid()                               = 0
getuid()                                = 0
getgid()                                = 0
access("/run/mount", R_OK|W_OK)         = 0
stat("/PoolB/daily.2020.12.13", {st_mode=S_IFDIR|0755, st_size=39, ...}) = 0
readlink("/PoolB", 0x7fffc79309b0, 4096) = -1 EINVAL (Invalid argument)
readlink("/PoolB/daily.2020.12.13", 0x7fffc79309b0, 4096) = -1 EINVAL (Invalid argument)
lstat("/etc/mtab", {st_mode=S_IFLNK|0777, st_size=17, ...}) = 0
open("/proc/self/mountinfo", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4088777000
read(3, "18 41 0:18 / /sys rw,nosuid,node"..., 1024) = 1024
stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", 0x7fffc7930370) = -1 ENOENT (No such file or directory)
read(3, "rfs efivarfs rw\n29 25 0:25 / /sy"..., 1024) = 1024
read(3, "ev,noexec,relatime shared:19 - c"..., 1024) = 1024
read(3, ",xattr,posixacl\n131 128 0:42 / /"..., 1024) = 1024
read(3, " - zfs PoolB/daily.2020.12.13 rw"..., 1024) = 145
readlink("/proc", 0x7fffc7930830, 4096) = -1 EINVAL (Invalid argument)
readlink("/proc/self", "37241", 4096)   = 5
readlink("/proc/37241", 0x7fffc7930830, 4096) = -1 EINVAL (Invalid argument)
readlink("/proc/37241/mountinfo", 0x7fffc7930830, 4096) = -1 EINVAL (Invalid argument)
read(3, "", 1024)                       = 0
close(3)                                = 0
munmap(0x7f4088777000, 4096)            = 0
getuid()                                = 0
geteuid()                               = 0
getgid()                                = 0
getegid()                               = 0
prctl(PR_GET_DUMPABLE)                  = 1 (SUID_DUMP_USER)
stat("/run", {st_mode=S_IFDIR|0755, st_size=960, ...}) = 0
stat("/run/mount/utab", 0x7fffc79328c0) = -1 ENOENT (No such file or directory)
stat("/sbin/umount.zfs", 0x7fffc7931a80) = -1 ENOENT (No such file or directory)
stat("/sbin/fs.d/umount.zfs", 0x7fffc7931a80) = -1 ENOENT (No such file or directory)
stat("/sbin/fs/umount.zfs", 0x7fffc7931a80) = -1 ENOENT (No such file or directory)
stat("/run/mount/utab", 0x7fffc7932b30) = -1 ENOENT (No such file or directory)
umount2("/PoolB/daily.2020.12.13", 0)   = -1 EBUSY (Device or resource busy)
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4088777000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2502
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7f4088777000, 4096)            = 0
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/util-linux.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
write(2, "umount: ", 8umount: )                 = 8
write(2, "/PoolB/daily.2020.12.13: target "..., 153/PoolB/daily.2020.12.13: target is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))) = 153
write(2, "\n", 1
)                       = 1
close(1)                                = 0
close(2)                                = 0
exit_group(32)                          = ?
+++ exited with 32 +++
[root@head-02 proc]# 

As you can see, the strace reveals the system call umount2 fails with EBUSY, which in the man page for that system call, just says "The Target is Busy".

We even deleted all the contents of that filesystem with rm -rf /PoolB/daily.2020.12.13/ which ran successfully. We don't have swap spaces on it, and it isn't listed in /etc/exports.

We are at a loss at this point, and for obvious reasons, rebooting the system is for all intents and purposes a non-starter. We should not have to reboot anyway just because a filesystem has to be unmounted. Any advice would be greatly appreciated.

Linux Kernel version is 3.10.0-1062.4.3.el7.x86_64 ZFS version is 0.8.2-1.el7 as is the zfs-dkms package.

Thank you for reading this far.

I have two CentOS 7 servers (Hypervisor and Relay for this scenario) . I wish to connect to a VNC port on Hypervisor which is behind a firewall by using Relay with is a server with SSH open to Hypervisor.

I ask Hypervisor to establish the connection with the command:

ssh -N -R 0.0.0.0:5912:127.0.0.1:5912 root@Relay

Once I do this, I can connect to the port on Relay by

telnet localhost:5912 

and I receive the response: RFB 003.008

When I telnet using the assigned IP address:

telnet 1.2.3.4:5912 

on Relay: Connection Refused

Note, the IP 1.2.3.4 is the IP of Relay's NIC, and not some NAT'd IP.

I can not connect from another server within Relay's subnet either.

I have a very strange issue that I have come across twice now. It is making me go cross-eyed.

We have two sites, one runs our server infrastructure and the other hosts end-users. There is an IPSEC tunnel established between StrongSWAN and Cisco ASA (v. 9.8.1). Connectivity is over a 100Mbps AT&T Fiber and Blanded Colo Bandwidth.

All devices and services are working well and we do not drop pings from almost all equipment. Pinging access points and thin clients works well.

One printer, an HP 401N LaserJet Printer will ping for 3 minutes (185 - 205 pings), and then stop (pinging from the remote side). Locally, pinging and services continue to work. All services become inaccessible from the remote side. The only way to restore connectivity is to change network settings (we have been setting the gateway to .2, then back to .1), or rebooting the printer.

The strange part is, its only the remote side that looses connectivity, and it only happens after a few minutes.

A similar issue came up a few months ago, but that was a different make / model printer, was over a decade old, and we determined it was too old to troubleshoot and replaced the printer. We have other printers that work just fine. It is only this printer that is affected.

We have tried disabling IPv6, turning off all ACLs on the printer's administrative page, and maintaining a constant pint (both from local and remote). 3 minutes does not correspond to the tunnel's re-keying timer, or anything else we can think of.

I have a ZFS system serving many VMs. In it, we installed 12x4TB SAS disks and configured them in a mirror. We added two PCI-E SSDs (Samsung 960s with 512GB each), but they are each consumed to just over 50% rather than caching all the data they can. We have plenty of consumed storage, yet the SSDs are only filling up to about half way.

System is a CentOS 7 box with ZFS on Linux version 0.6.5.9-1.el7_3.centos.

Here is a snapshot of zpool iostat:

                                                      capacity     operations    bandwidth
pool                                               alloc   free   read  write   read  write
-------------------------------------------------  -----  -----  -----  -----  -----  -----
stgpool                                            13.3T  8.46T     26   1020   482K  10.6M
  mirror                                           2.21T  1.41T      4    165  50.5K  1.79M
    S1D0                                               -      -      3    158  52.0K  2.05M
    S2D0                                               -      -      0    158  11.2K  2.05M
  mirror                                           2.21T  1.41T      2    184  51.7K  1.97M
    S3D0                                               -      -      2    177  47.2K  2.26M
    S4D0                                               -      -      0    177  13.6K  2.26M
  mirror                                           2.21T  1.41T      3    170   104K  1.66M
    S1D1                                               -      -      2    169  90.4K  1.94M
    S2D1                                               -      -      1    169  21.6K  1.94M
  mirror                                           2.21T  1.41T      5    142   129K  1.61M
    S3D1                                               -      -      3    139   109K  1.86M
    S4D1                                               -      -      1    139  32.8K  1.86M
  mirror                                           2.21T  1.41T      4    170  93.7K  1.81M
    S1D2                                               -      -      4    162  95.2K  2.11M
    S2D2                                               -      -      0    162  11.2K  2.11M
  mirror                                           2.21T  1.41T      4    186  54.2K  1.78M
    S3D2                                               -      -      3    175  40.0K  2.10M
    S4D2                                               -      -      0    175  27.2K  2.10M
logs                                                   -      -      -      -      -      -
  ata-Samsung_SSD_850_EVO_250GB_S21NNXAG918721R        0   232G      0      0      0      0
  ata-Samsung_SSD_850_EVO_250GB_S21NNXAGA59337A        0   232G      0      0      0      0
  ata-Samsung_SSD_850_EVO_250GB_S21NNXAGA69590F        0   232G      0      0      0      0
cache                                                  -      -      -      -      -      -
  nvme-Samsung_SSD_960_PRO_512GB_S3EWNCAHC01880D    266G   211G     38     81   556K  2.85M
  nvme-Samsung_SSD_960__PRO_512GB_S3EWNCAHB04288W   266G   211G     37     56   416K  2.10M
-------------------------------------------------  -----  -----  -----  -----  -----  -----

Zpool settings:

NAME     PROPERTY                    VALUE                       SOURCE
stgpool  size                        21.8T                       -
stgpool  capacity                    61%                         -
stgpool  altroot                     -                           default
stgpool  health                      ONLINE                      -
stgpool  guid                        1784205276891874933         default
stgpool  version                     -                           default
stgpool  bootfs                      -                           default
stgpool  delegation                  on                          default
stgpool  autoreplace                 off                         default
stgpool  cachefile                   -                           default
stgpool  failmode                    wait                        default
stgpool  listsnapshots               off                         default
stgpool  autoexpand                  off                         default
stgpool  dedupditto                  0                           default
stgpool  dedupratio                  1.00x                       -
stgpool  free                        8.46T                       -
stgpool  allocated                   13.3T                       -
stgpool  readonly                    off                         -
stgpool  ashift                      0                           default
stgpool  comment                     -                           default
stgpool  expandsize                  -                           -
stgpool  freeing                     0                           default
stgpool  fragmentation               58%                         -
stgpool  leaked                      0                           default
stgpool  feature@async_destroy       enabled                     local
stgpool  feature@empty_bpobj         active                      local
stgpool  feature@lz4_compress        active                      local
stgpool  feature@spacemap_histogram  active                      local
stgpool  feature@enabled_txg         active                      local
stgpool  feature@hole_birth          active                      local
stgpool  feature@extensible_dataset  enabled                     local
stgpool  feature@embedded_data       active                      local
stgpool  feature@bookmarks           enabled                     local
stgpool  feature@filesystem_limits   enabled                     local
stgpool  feature@large_blocks        enabled                     local

arcstat.py info:

    time  read  l2read  hit%  hits  miss%  miss  l2hit%  l2miss%  arcsz     c  l2size  
01:44:35  2.1K     268    87  1.8K     12   268       2       97   125G  125G    778G  
01:44:36  6.5K     583    91  6.0K      8   583       1       98   125G  125G    778G  
01:44:37  1.1K     277    75   835     24   277       6       93   125G  125G    778G  
01:44:38  1.5K     230    84  1.3K     15   230       1       98   125G  125G    778G  
01:44:39  1.8K     141    91  1.6K      8   141       2       97   125G  125G    778G  
01:44:40  1.4K     203    85  1.2K     14   203      19       80   125G  125G    778G  
01:44:41  4.0K     291    92  3.7K      7   291      11       88   125G  125G    778G  
01:44:42  1.7K      95    94  1.6K      5    95       8       91   125G  125G    778G  
01:44:43  1.1K      84    92  1.0K      7    84       2       97   125G  125G    778G  

I am trying to maximize my read rates and consume the 960's capacity. Any help is appreciated :)

Im having an issue which I have a really hard time debugging. Running ZFS, by system "hiccuped", dumped some information into DMESG, and continued working.

My ZFS is hosting VMs on ESXi. When this issue occurs, many of the VMs experience block IO errors, and some of them drop into read-only mode, requiring restores from backup or fsck to repair the filesystems. This issue only occurs very occasionally, and I have hammered the system, trying to stress it out, it does not seem to be performance related. Only occurs every few months, so conclusively solving it seems to be a pipe-dream to me.

First off, some info about my system (Centos 7, 4.5).

[root@zfs-head ~]# name -a

Linux zfs-head 4.5.0-1.el7.elrepo.x86_64 #1 SMP Mon Mar 14 10:24:58 EDT 2016 x86_64 x86_64 x86_64 GNU/Linux

dmesg entries:

[4331253.022999] sd 2:0:28:0: [sdaa] tag#2 CDB: Read(10) 28 00 10 a8 3d b5 00 00 20 00
[4331253.023006] mpt3sas_cm0:   sas_address(0x5000c500837f31f2), phy(8)
[4331253.023008] mpt3sas_cm0:   enclosure_logical_id(0x50010c60004d41ff),slot(0)
[4331253.023010] mpt3sas_cm0:   enclosure level(0x0003), connector name(     )
[4331253.023013] mpt3sas_cm0:   handle(0x002d), ioc_status(scsi data underrun)(0x0045), smid(222)
[4331253.023016] mpt3sas_cm0:   request_len(131072), underflow(16384), resid(131072)
[4331253.023018] mpt3sas_cm0:   tag(0), transfer_count(0), sc->result(0x00000000)
[4331253.023020] mpt3sas_cm0:   scsi_status(check condition)(0x02), scsi_state(autosense valid )(0x01)
[4331253.023023] mpt3sas_cm0:   [sense_key,asc,ascq]: [0x06,0x2a,0x01], count(96)
[4331253.023030] sd 2:0:28:0: Mode parameters changed
[4331266.475222] sd 2:0:29:0: [sdab] tag#29 CDB: Write(10) 2a 00 09 97 6e c1 00 00 02 00
[4331266.475229] mpt3sas_cm0:   sas_address(0x5000c500837f25c6), phy(9)
[4331266.475232] mpt3sas_cm0:   enclosure_logical_id(0x50010c60004d41ff),slot(1)
[4331266.475234] mpt3sas_cm0:   enclosure level(0x0003), connector name(     )
[4331266.475237] mpt3sas_cm0:   handle(0x002e), ioc_status(scsi data underrun)(0x0045), smid(139)
[4331266.475239] mpt3sas_cm0:   request_len(8192), underflow(1024), resid(8192)
[4331266.475241] mpt3sas_cm0:   tag(0), transfer_count(0), sc->result(0x00000000)
[4331266.475244] mpt3sas_cm0:   scsi_status(check condition)(0x02), scsi_state(autosense valid )(0x01)
[4331266.475246] mpt3sas_cm0:   [sense_key,asc,ascq]: [0x06,0x2a,0x01], count(96)
[4331266.475252] sd 2:0:29:0: Mode parameters changed

pool status:

[root@zfs-head ~]# pool status
  pool: storage
 state: ONLINE
  scan: none requested
config:

    NAME                                             STATE     READ WRITE CKSUM
    storage                                          ONLINE       0     0     0
      mirror-0                                       ONLINE       0     0     0
        s1d1                                         ONLINE       0     0     0
        s2d1                                         ONLINE       0     0     0
      mirror-1                                       ONLINE       0     0     0
        s3d1                                         ONLINE       0     0     0
        s4d1                                         ONLINE       0     0     0
      mirror-2                                       ONLINE       0     0     0
        s1d2                                         ONLINE       0     0     0
        s2d2                                         ONLINE       0     0     0
      mirror-3                                       ONLINE       0     0     0
        s3d2                                         ONLINE       0     0     0
        s4d2                                         ONLINE       0     0     0
      mirror-4                                       ONLINE       0     0     0
        s1d3                                         ONLINE       0     0     0
        s2d3                                         ONLINE       0     0     0
      mirror-5                                       ONLINE       0     0     0
        s3d3                                         ONLINE       0     0     0
        s4d3                                         ONLINE       0     0     0
    logs
      ata-Samsung_SSD_850_PRO_128GB_S24ZNXAGA10768M  ONLINE       0     0     0
    cache
      ata-Samsung_SSD_850_EVO_250GB_S21NNXAG918721R  ONLINE       0     0     0
      ata-Samsung_SSD_850_EVO_250GB_S21NNXAGA59337A  ONLINE       0     0     0
      ata-Samsung_SSD_850_EVO_250GB_S21NNXAGA69590F  ONLINE       0     0     0

errors: No known data errors
[root@zfs-head ~]# 

My Vdev map:

[root@zfs-head ~]# cat /etc/zfs/vdev_id.conf
#     by-vdev
#     name     fully qualified or base name of device link
alias s1d1       /dev/disk/by-id/scsi-35000c500837ff247
alias s1d2       /dev/disk/by-id/scsi-35000c500837f15c3
alias s1d3       /dev/disk/by-id/scsi-35000c500837f137f
alias s2d1       /dev/disk/by-id/scsi-35000c500837f377b
alias s2d2       /dev/disk/by-id/scsi-35000c500837f5bf7
alias s2d3       /dev/disk/by-id/scsi-35000c500837f75bf
alias s3d1       /dev/disk/by-id/scsi-35000c500837f14d3
alias s3d2       /dev/disk/by-id/scsi-35000c500837f571b
alias s3d3       /dev/disk/by-id/scsi-35000c500837f604f
alias s4d1       /dev/disk/by-id/scsi-35000c500837f31f3
alias s4d2       /dev/disk/by-id/scsi-35000c500837f25c7
alias s4d3       /dev/disk/by-id/scsi-35000c500837f14cf

[root@zfs-head ~]# 

The box didn't restart, or really even acknowledge that there was an issue, save for the dmesg entries. I have googled those entries to my level best, but did not find anything relevant.

Help appreciated!

When using CENTOS 7 with Kernel 4.5, I can not yum install kernel-headers with the same kernel version. I am stuck on 3.10, which is what Centos comes with. How can I install Kernel-headers 4.5 and Kernel-devel which is needed for compiling drivers?

The packages are not in a repo with the mainline kernel updates:

[root@localhost tmp]# yum list | grep kernel
kernel-ml.x86_64                        4.5.0-1.el7.elrepo             @elrepo-kernel
abrt-addon-kerneloops.x86_64            2.1.11-36.el7.centos           base     
erlang-kernel.x86_64                    R16B-03.16.el7                 epel     
kernel.x86_64                           3.10.0-327.13.1.el7            updates  
kernel-abi-whitelists.noarch            3.10.0-327.13.1.el7            updates  
kernel-debug.x86_64                     3.10.0-327.13.1.el7            updates  
kernel-debug-devel.x86_64               3.10.0-327.13.1.el7            updates  
kernel-devel.x86_64                     3.10.0-327.13.1.el7            updates  
kernel-doc.noarch                       3.10.0-327.13.1.el7            updates  
kernel-headers.x86_64                   3.10.0-327.13.1.el7            updates  
kernel-tools.x86_64                     3.10.0-327.13.1.el7            updates  
kernel-tools-libs.x86_64                3.10.0-327.13.1.el7            updates  
kernel-tools-libs-devel.x86_64          3.10.0-327.13.1.el7            updates  
libreport-plugin-kerneloops.x86_64      2.1.11-32.el7.centos           base     
lirc-disable-kernel-rc.x86_64           0.9.1a-4.el7                   epel     
php-symfony-http-kernel.noarch          2.7.10-1.el7                   epel     
texlive-l3kernel.noarch                 2:svn29409.SVN_4469-38.el7     base     
texlive-l3kernel-doc.noarch             2:svn29409.SVN_4469-38.el7     base     
[root@localhost tmp]# uname -a
Linux localhost.localdomain 4.5.0-1.el7.elrepo.x86_64 #1 SMP Mon Mar 14 10:24:58 EDT 2016 x86_64 x86_64 x86_64 GNU/Linux
[root@localhost tmp]# 

In Linux, I would like to create a file where the contents of the file are dynamically generated by the output of a program.

The examples which already exist are /dev/random /dev/urandom, /dev/zero, etc...

I would like to create a file such as /home/sam/currentdate.txt where by the contents of the file are the result of running a command (such as date, and would read:

[sam@myserver ~]$ cat ./currentdate.txt
Wed Dec  2 12:12:09 PST 2015
[sam@myserver ~]$ 

For my use case, I need to track the output of certain programs by another program which can not read the contents of a program's output, but rather the contents of a file. I can save the contents of the output via a script by using date > filename.txt but that requires saving the output to disk first, which is going to be inefficient in my use case.

I have shared storage which is mounted (read-only) on SERVER1, and mounted (read-write) on SERVER2.

SERVER2 is writing data to a file. SERVER1 is reading data from that file.

The file poling is <100ms, which is fast, but lets just say it's necessary.

SERVER1 is accessing the filesystem cache, and not showing new data in the file. I can drop_caches which then returns the latest data in that file, but this process drops all file caches on SERVER1, which is not what I want to do, and it takes too long to process.

I have tried using the cat command, as well as dd if=/path/to/file -iflag=direct, but that doesn't work either. It returns an error. The flag seems not to be supported for files. Any idea on how to read a file directly from a disk, or how to disable filesystem cache for a file/directory/mount point?

I would also consider disabling file cache system-wide, but thats not my first preference, this is just for the one file. Google doesn't return a way to do this either, as most forum posts argue against such action.

Thanks!

I have a DNS server behind a Cisco ASA firewall. The Cisco firewall is performing NAT. When inside the network, I can NSLOOKUP directly to the server. When the client is external, requests time out.

Assuming the internal IP Address of the DNS Server is 10.0.0.10 and the public IP Address is 1.2.3.4:

My relevant configs:

object network NS1_SERVVE_COM_dns
 host 10.0.0.10

access-list outside_in extended permit tcp any object NS1_SERVVE_COM_dns eq domain 

object network NS1_SERVVE_COM_dns
 nat (inside,outside) static 1.2.3.4 service tcp domain domain 

Also I entered the following to allow Active Directory to resolve internet names:

policy-map type inspect dns preset_dns_map
 parameters
  message-length maximum client auto
  message-length maximum 8192
policy-map global_policy
 class inspection_default
  inspect dns preset_dns_map 

I have tried to change TCP to UDP, but the result is the same. Any thoughts?

firewall# packet-tracer input outside tcp 4.2.2.1 domain 1.2.3.4 domai$

Phase: 1
Type: UN-NAT
Subtype: static
Result: ALLOW
Config:
object network NS1_SERVVE_COM_dns
 nat (inside,outside) static 1.2.3.4 service tcp domain domain 
Additional Information:
NAT divert to egress interface inside
Untranslate 1.2.3.4/53 to 10.0.0.10/53

Phase: 2
Type: ACCESS-LIST
Subtype: log
Result: ALLOW
Config:
access-group outside_in in interface outside
access-list outside_in extended permit tcp any object NS1_SERVVE_COM_dns eq domain 
Additional Information:

Phase: 3
Type: NAT
Subtype: per-session
Result: ALLOW
Config:       
Additional Information:

Phase: 4
Type: IP-OPTIONS
Subtype: 
Result: ALLOW
Config:
Additional Information:

Phase: 5
Type: NAT
Subtype: rpf-check
Result: ALLOW
Config:
object network NS1_SERVVE_COM_dns
 nat (inside,outside) static 1.2.3.4 service tcp domain domain 
Additional Information:

Phase: 6
Type: NAT
Subtype: per-session
Result: ALLOW
Config:
Additional Information:

Phase: 7
Type: IP-OPTIONS
Subtype: 
Result: ALLOW
Config:
Additional Information:

Phase: 8
Type: FLOW-CREATION
Subtype: 
Result: ALLOW
Config:
Additional Information:
New flow created with id 7845414, packet dispatched to next module

Result:
input-interface: outside
input-status: up
input-line-status: up
output-interface: inside
output-status: up
output-line-status: up
Action: allow 

When experiencing write I/O, the log column in zpool iostat -v does not show any ZIL activity, ever. This results in higher than expected wait times when writing data to disk (sometimes over 80ms during contention).

                     capacity     operations    bandwidth
    pool              alloc   free   read  write   read  write
----------------  -----  -----  -----  -----  -----  -----  
storage           1.88T  2.09T      3  3.01K   512K  39.3M
  mirror           961G  1.05T      0  1.97K   128K  20.8M
    mpathf            -      -      0    393      0  20.8M
    mpathg            -      -      0    391   128K  20.6M
  mirror           961G  1.05T      2  1.04K   384K  18.5M
    mpathi            -      -      1    379   256K  21.1M
    mpathj            -      -      0    281   128K  18.3M
logs                  -      -      -      -      -      -
  /zlog/zilcache      0  15.9G      0      0      0      0
cache                 -      -      -      -      -      -
  mpathk           232G     8M      1      0   130K      0
  mpathl           232G     8M      1      0   130K      0
----------------  -----  -----  -----  -----  -----  -----

My /zlog/zilcache device never has any IO. It is a file on very fast flash. I can write and read it when I remove it from the ZFS store, but ZFS seems to ignore it.

The device looks available:

  pool: storage
 state: ONLINE
  scan: scrub repaired 0 in 19h31m with 0 errors on Wed Nov 19 07:39:03 2014
config:

    NAME              STATE     READ WRITE CKSUM
    storage           ONLINE       0     0     0
      mirror-0        ONLINE       0     0     0
        mpathf        ONLINE       0     0     0
        mpathg        ONLINE       0     0     0
      mirror-1        ONLINE       0     0     0
        mpathi        ONLINE       0     0     0
        mpathj        ONLINE       0     0     0
    logs
      /zlog/zilcache  ONLINE       0     0     0
    cache
      mpathk          ONLINE       0     0     0
      mpathl          ONLINE       0     0     0

errors: No known data errors

Any way to configure ZFS to cache writes to the logs device for faster acknowledgements?

Thanks

I have an ASA 5515 as my internet firewall. It is not allowing me to do NS Lookups from any internal DNS Servers, or clients. If I set my nslookup server to 8.8.8.8 (google DNS), I can resolve public DNS names. If I am on the internal network, breaks.

I have the following in my ASA:

policy-map type inspect dns preset_dns_map
 parameters
  message-length maximum client auto
  message-length maximum 8192
policy-map global_policy
 class inspection_default
  inspect ftp 
  inspect h323 h225 
  inspect h323 ras 
  inspect ip-options 
  inspect netbios 
  inspect rsh 
  inspect rtsp 
  inspect skinny  
  inspect esmtp 
  inspect sqlnet 
  inspect sunrpc 
  inspect tftp 
  inspect sip  
  inspect xdmcp 
  inspect pptp 
  inspect ipsec-pass-thru 
  inspect icmp 
  inspect dns preset_dns_map 

Any ideas as to why its not working?