HOWTO - jails - FreeBSD 12 + VNET + ZFS

From: BulkMailForRudy <crapsh_at_monkeybrains.net>
Date: Fri, 25 Jan 2019 10:33:29 -0800
I love using jails.  For many years, I used a tool to help out: ezjail, 
now I am just raw-dogging it by using the config file in /etc/jail.conf


Here is my config:

# /etc/jail.conf
# VNET is used to send an epair to each jail.
# The epair is renamed jail0 with exec.created in each jail.
# exec.prestrt Script creates bridge0 if needed.

# Global settings applied to all jails.

# haven't found a good reason to run a jail as NOT root
exec.system_user  = "root";
exec.jail_user    = "root";
mount.devfs;
allow.raw_sockets;
devfs_ruleset     = "5";

# Networking and the exec cycle
$uplinkdev        = "ix0";
vnet;
vnet.interface    = "jail0";               # default vnet interface
exec.prestart     = "ifconfig bridge0 > /dev/null 2> /dev/null || ( 
ifconfig bridge0 create up && ifconfig bridge0 addm $uplinkdev )";
exec.prestart    += "ifconfig $epair create up                 || echo 
'Skipped creating epair (exists?)'";
exec.prestart    += "ifconfig bridge0 addm ${epair}a           || echo 
'Skipped adding bridge member (already member?)''";
exec.created      = "ifconfig ${epair}b name jail0             || echo 
'Skipped renaming ifdev to jail0'";
exec.clean;
exec.start        = "/bin/sh /etc/rc";
exec.stop         = "/bin/sh /etc/rc.shutdown";
exec.poststop     = "ifconfig bridge0 deletem ${epair}a";
#exec.poststop    += "ifconfig ${epair}a destroy";

# Per-jail settings
ns1 {
     path          = "/data/ns1.monkeybrains.net/";
     host.hostname = "ns1.monkeybrains.net";
     $epair        = "epair0";  # must be unique in every jail
}

tac {
     path          = "/data/tac.monkeybrains.net/";
     host.hostname = "tac.monkeybrains.net";
     $epair        = "epair1";
}


=====================================

Here is a look at ifconfig before and after jail creation.


============  Before jails start up ============

ix0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=e53fbb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6>
     ether ac:1f:6b:6a:14:78
     inet 10.1.2.3 netmask 0xffffff00 broadcast 10.1.2.255
     inet6 fe80::ae1f:aaaa:aaaa:1478%ix0 prefixlen 64 scopeid 0x1
     inet6 2607:f598::a:a prefixlen 64
     media: Ethernet autoselect (1000baseT <full-duplex>)
     status: active
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
     inet6 ::1 prefixlen 128
     inet6 fe80::1%lo0 prefixlen 64 scopeid 0x3
     inet 127.0.0.1 netmask 0xff000000
     groups: lo

ix0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 
mtu 1500
options=a538b9<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6>
     ether ac:1f:6b:6a:14:78
     inet 208.69.40.26 netmask 0xffffff00 broadcast 208.69.40.255
     inet6 fe80::ae1f:6bff:fe6a:1478%ix0 prefixlen 64 scopeid 0x1
     inet6 2607:f598::d045:281a prefixlen 64
     media: Ethernet autoselect (1000baseT <full-duplex>)
     status: active
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
ix1: flags=8802<BROADCAST,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=e53fbb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6>
     ether ac:1f:6b:6a:14:79
     media: Ethernet autoselect
     status: no carrier
     nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
     inet6 ::1 prefixlen 128
     inet6 fe80::1%lo0 prefixlen 64 scopeid 0x3
     inet 127.0.0.1 netmask 0xff000000
     groups: lo
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 
1500
     ether 02:16:09:1c:af:00
     id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
     maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
     root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
     member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
             ifmaxaddr 0 port 6 priority 128 path cost 2000
     member: epair0a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
             ifmaxaddr 0 port 5 priority 128 path cost 2000
     member: ix0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
             ifmaxaddr 0 port 1 priority 128 path cost 2000
     groups: bridge
     nd6 options=1<PERFORMNUD>
epair0a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> 
metric 0 mtu 1500
     options=8<VLAN_MTU>
     ether 02:8d:76:e8:34:0a
     inet6 fe80::8d:76ff:fee8:340a%epair0a prefixlen 64 scopeid 0x5
     groups: epair
     media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
     status: active
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> 
metric 0 mtu 1500
     options=8<VLAN_MTU>
     ether 02:7a:d1:7c:f8:0a
     inet6 fe80::7a:d1ff:fe7c:f80a%epair1a prefixlen 64 scopeid 0x6
     groups: epair
     media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
     status: active
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>



============  Start up jails ============

# service jail start
Starting jails: ns1 tac.

# ifconfig

ix0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 
mtu 1500
options=a538b9<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6>
     ether ac:1f:6b:6a:14:78
     inet 10.1.2.3 netmask 0xffffff00 broadcast 10.1.2.255
     inet6 fe80::ae1f:aaaa:aaaa:1478%ix0 prefixlen 64 scopeid 0x1
     inet6 2607:f598::a:a prefixlen 64
     media: Ethernet autoselect (1000baseT <full-duplex>)
     status: active
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
     inet6 ::1 prefixlen 128
     inet6 fe80::1%lo0 prefixlen 64 scopeid 0x3
     inet 127.0.0.1 netmask 0xff000000
     groups: lo
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 
1500
     ether 02:16:09:1c:af:00
     id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
     maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
     root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
     member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
             ifmaxaddr 0 port 6 priority 128 path cost 2000
     member: epair0a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
             ifmaxaddr 0 port 5 priority 128 path cost 2000
     member: ix0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
             ifmaxaddr 0 port 1 priority 128 path cost 2000
     groups: bridge
     nd6 options=1<PERFORMNUD>
epair0a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> 
metric 0 mtu 1500
     options=8<VLAN_MTU>
     ether 02:8d:76:e8:34:0a
     inet6 fe80::8d:76ff:fee8:340a%epair0a prefixlen 64 scopeid 0x5
     groups: epair
     media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
     status: active
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> 
metric 0 mtu 1500
     options=8<VLAN_MTU>
     ether 02:7a:d1:7c:f8:0a
     inet6 fe80::7a:d1ff:fe7c:f80a%epair1a prefixlen 64 scopeid 0x6
     groups: epair
     media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
     status: active
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>


# jls
    JID  IP Address      Hostname                      Path
     19                  ns1.monkeybrains.net /data/ns1.monkeybrains.net

     20                  tac.monkeybrains.net /data/tac.monkeybrains.net


# jexec ns1 ifconfig
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
     inet6 ::1 prefixlen 128
     inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
     inet 127.0.0.1 netmask 0xff000000
     groups: lo
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
jail0: flags=8842<BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
     options=8<VLAN_MTU>
     ether 02:8d:76:e8:34:0b
     groups: epair
     media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
     status: active
     nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
# jexec tac ifconfig
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
     inet6 ::1 prefixlen 128
     inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
     inet 127.0.0.1 netmask 0xff000000
     groups: lo
     nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
jail0: flags=8842<BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
     options=8<VLAN_MTU>
     ether 02:7a:d1:7c:f8:0b
     groups: epair
     media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
     status: active
     nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
vlan91: flags=8003<UP,BROADCAST,MULTICAST> metric 0 mtu 1500
     ether 00:00:00:00:00:00
     groups: vlan
     vlan: 0 vlanpcp: 0 parent interface: <none>
     nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
Received on Fri Jan 25 2019 - 17:52:32 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:41:20 UTC