❯ go version
go version go1.22.2 linux/amd64
❯ go env GOPATH
/home/kkumtree/go
❯ go install sigs.k8s.io/[email protected]
go: downloading sigs.k8s.io/kind v0.24.0
go: downloading github.com/spf13/pflag v1.0.5
go: downloading github.com/alessio/shellescape v1.4.2
go: downloading github.com/spf13/cobra v1.8.0
go: downloading github.com/pkg/errors v0.9.1
go: downloading github.com/mattn/go-isatty v0.0.20
go: downloading golang.org/x/sys v0.6.0
go: downloading github.com/pelletier/go-toml v1.9.5
go: downloading github.com/BurntSushi/toml v1.4.0
go: downloading github.com/evanphx/json-patch/v5 v5.6.0
go: downloading gopkg.in/yaml.v3 v3.0.1
go: downloading sigs.k8s.io/yaml v1.4.0
go: downloading github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2
❯ vi .profile # 동적 지정하는 것으로 자세한건 이전 포스팅 참조  
❯ source .profile
❯ kind version
kind v0.24.0 go1.22.2 linux/amd64

cat <<EOT> kind-metallb-test.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
featureGates:
  "InPlacePodVerticalScaling": true
  "MultiCIDRServiceAllocator": true
nodes:
- role: control-plane
  labels:
    mynode: control-plane
    topology.kubernetes.io/zone: ap-northeast-2a
  extraPortMappings:
  - containerPort: 30000
    hostPort: 30000
  - containerPort: 30001
    hostPort: 30001
  - containerPort: 30002
    hostPort: 30002
  - containerPort: 30003
    hostPort: 30003
  - containerPort: 30004
    hostPort: 30004
  kubeadmConfigPatches:
  - |
    kind: ClusterConfiguration
    apiServer:
      extraArgs:
        runtime-config: api/all=true
    controllerManager:
      extraArgs:
        bind-address: 0.0.0.0
    etcd:
      local:
        extraArgs:
          listen-metrics-urls: http://0.0.0.0:2381
    scheduler:
      extraArgs:
        bind-address: 0.0.0.0
  - |
    kind: KubeProxyConfiguration
    metricsBindAddress: 0.0.0.0
- role: worker
  labels:
    mynode: worker1
    topology.kubernetes.io/zone: ap-northeast-2a
- role: worker
  labels:
    mynode: worker2
    topology.kubernetes.io/zone: ap-northeast-2b
- role: worker
  labels:
    mynode: worker3
    topology.kubernetes.io/zone: ap-northeast-2c
networking:
  podSubnet: 10.10.0.0/16
  serviceSubnet: 10.200.1.0/24
EOT

kind create cluster --config kind-metallb-test.yaml --name myk8s --image kindest/node:v1.31.0
# Install additional tools 
docker exec -it myk8s-control-plane sh -c 'apt update && apt install tree psmisc lsof wget bsdmainutils bridge-utils net-tools dnsutils ipset ipvsadm nfacct tcpdump ngrep iputils-ping arping git vim arp-scan -y'

# cidr check
❯ kubectl cluster-info dump | grep -m 2 -E "cluster-cidr|service-cluster-ip-range"
                            "--service-cluster-ip-range=10.200.1.0/24",
                            "--cluster-cidr=10.10.0.0/16",
# confirm kube-proxy mode: iptables proxy mode
❯ kubectl describe  configmap -n kube-system kube-proxy | grep mode
mode: iptables
# iptables info  
# 출력값은 너무 길어서 생략 / MetalLB 설치 후 대조용  
for i in filter nat mangle raw ; do echo ">> IPTables Type : $i <<"; docker exec -it myk8s-control-plane  iptables -t $i -S ; echo; done
for i in filter nat mangle raw ; do echo ">> IPTables Type : $i <<"; docker exec -it myk8s-worker  iptables -t $i -S ; echo; done
for i in filter nat mangle raw ; do echo ">> IPTables Type : $i <<"; docker exec -it myk8s-worker2 iptables -t $i -S ; echo; done
for i in filter nat mangle raw ; do echo ">> IPTables Type : $i <<"; docker exec -it myk8s-worker3 iptables -t $i -S ; echo; done

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: webpod1
  labels:
    app: webpod
spec:
  nodeName: myk8s-worker
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Pod
metadata:
  name: webpod2
  labels:
    app: webpod
spec:
  nodeName: myk8s-worker2
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
EOF

# pod/webpod1 created
# pod/webpod2 created

git clone https://github.com/metallb/metallb-operator.git
cd metallb-operator
make deploy
cat << EOF | kubectl apply -f -
apiVersion: metallb.io/v1beta1
kind: MetalLB
metadata:
  name: metallb
  namespace: metallb-system
EOF
make test
make test-e2e

# 아래 커맨드 응용
# kubectl apply -f bin/metallb-operator.yaml 
# CRD (및 오퍼레이터 배포 커맨드) 다운로드  
curl -LO https://raw.githubusercontent.com/metallb/metallb-operator/refs/heads/main/bin/metallb-operator.yaml 
# CRD 적용. 확인해보니 오퍼레이터도 함께 배포하는 모양  
kubectl apply -f metallb-operator.yaml

❯ curl -LO https://raw.githubusercontent.com/metallb/metallb-operator/refs/heads/main/bin/metallb-operator.yaml 
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  233k  100  233k    0     0   799k      0 --:--:-- --:--:-- --:--:--  797k
❯ kubectl apply -f metallb-operator.yaml
namespace/metallb-system created
customresourcedefinition.apiextensions.k8s.io/bfdprofiles.metallb.io created
customresourcedefinition.apiextensions.k8s.io/bgpadvertisements.metallb.io created
customresourcedefinition.apiextensions.k8s.io/bgppeers.metallb.io created
customresourcedefinition.apiextensions.k8s.io/communities.metallb.io created
customresourcedefinition.apiextensions.k8s.io/frrconfigurations.frrk8s.metallb.io created
customresourcedefinition.apiextensions.k8s.io/frrnodestates.frrk8s.metallb.io created
customresourcedefinition.apiextensions.k8s.io/ipaddresspools.metallb.io created
customresourcedefinition.apiextensions.k8s.io/l2advertisements.metallb.io created
customresourcedefinition.apiextensions.k8s.io/metallbs.metallb.io created
customresourcedefinition.apiextensions.k8s.io/servicel2statuses.metallb.io created
serviceaccount/manager-account created
role.rbac.authorization.k8s.io/metallb-manager-role created
clusterrole.rbac.authorization.k8s.io/metallb-manager-role created
rolebinding.rbac.authorization.k8s.io/metallb-manager-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/metallb-manager-rolebinding created
secret/metallb-operator-webhook-server-cert created
secret/metallb-webhook-cert created
service/metallb-operator-webhook-service created
service/metallb-webhook-service created
deployment.apps/metallb-operator-controller-manager created
deployment.apps/metallb-operator-webhook-server created
validatingwebhookconfiguration.admissionregistration.k8s.io/metallb-operator-webhook-configuration created
validatingwebhookconfiguration.admissionregistration.k8s.io/metallb-webhook-configuration created
serviceaccount/controller created
serviceaccount/frr-k8s-daemon created
serviceaccount/speaker created
role.rbac.authorization.k8s.io/controller created
role.rbac.authorization.k8s.io/frr-k8s-daemon-role created
role.rbac.authorization.k8s.io/frr-k8s-daemon-scc created
role.rbac.authorization.k8s.io/pod-lister created
role.rbac.authorization.k8s.io/speaker created
clusterrole.rbac.authorization.k8s.io/metallb-system:kube-rbac-proxy created
clusterrole.rbac.authorization.k8s.io/frr-k8s-daemon-role created
clusterrole.rbac.authorization.k8s.io/frr-k8s-metrics-reader created
clusterrole.rbac.authorization.k8s.io/frr-k8s-proxy-role created
clusterrole.rbac.authorization.k8s.io/metallb-system:controller created
clusterrole.rbac.authorization.k8s.io/metallb-system:speaker created
rolebinding.rbac.authorization.k8s.io/controller created
rolebinding.rbac.authorization.k8s.io/frr-k8s-daemon-rolebinding created
rolebinding.rbac.authorization.k8s.io/frr-k8s-daemon-scc-binding created
rolebinding.rbac.authorization.k8s.io/pod-lister created
rolebinding.rbac.authorization.k8s.io/speaker created
clusterrolebinding.rbac.authorization.k8s.io/kube-rbac-proxy created
clusterrolebinding.rbac.authorization.k8s.io/frr-k8s-daemon-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/frr-k8s-proxy-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/metallb-system:controller created
clusterrolebinding.rbac.authorization.k8s.io/metallb-system:speaker created

❯ kubectl get crd | grep metallb
bfdprofiles.metallb.io                2024-10-02T14:18:46Z
bgpadvertisements.metallb.io          2024-10-02T14:18:46Z
bgppeers.metallb.io                   2024-10-02T14:18:46Z
communities.metallb.io                2024-10-02T14:18:46Z
frrconfigurations.frrk8s.metallb.io   2024-10-02T14:18:46Z
frrnodestates.frrk8s.metallb.io       2024-10-02T14:18:46Z
ipaddresspools.metallb.io             2024-10-02T14:18:46Z
l2advertisements.metallb.io           2024-10-02T14:18:46Z
metallbs.metallb.io                   2024-10-02T14:18:46Z
servicel2statuses.metallb.io          2024-10-02T14:18:46Z

❯ kubectl get all,configmap,secret,ep -n metallb-system
NAME                                                       READY   STATUS    RESTARTS   AGE
pod/metallb-operator-controller-manager-5dbc8fd577-bgczj   1/1     Running   0          13m
pod/metallb-operator-webhook-server-77d47cb764-9lcs8       1/1     Running   0          13m

NAME                                       TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)   AGE
service/metallb-operator-webhook-service   ClusterIP   10.200.1.159   <none>        443/TCP   13m
service/metallb-webhook-service            ClusterIP   10.200.1.149   <none>        443/TCP   13m

NAME                                                  READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/metallb-operator-controller-manager   1/1     1            1           13m
deployment.apps/metallb-operator-webhook-server       1/1     1            1           13m

NAME                                                             DESIRED   CURRENT   READY   AGE
replicaset.apps/metallb-operator-controller-manager-5dbc8fd577   1         1         1       13m
replicaset.apps/metallb-operator-webhook-server-77d47cb764       1         1         1       13m

NAME                         DATA   AGE
configmap/kube-root-ca.crt   1      13m

NAME                                          TYPE     DATA   AGE
secret/metallb-operator-webhook-server-cert   Opaque   4      13m
secret/metallb-webhook-cert                   Opaque   4      13m

NAME                                         ENDPOINTS        AGE
endpoints/metallb-operator-webhook-service   10.10.2.2:9443   13m
endpoints/metallb-webhook-service            10.10.3.3:9443   13m

❯ kubectl get pods -n metallb-system -l app=metallb -o jsonpath="{range .items[*]}{.metadata.name}{':\n'}{range .spec.containers[*]}{'  '}{.name}{' -> '}{.image}{'\n'}{end}{end}"
metallb-operator-webhook-server-77d47cb764-9lcs8:
  webhook-server -> quay.io/metallb/controller:main

❯ kubectl get ds,deploy -n metallb-system
NAME                                                  READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/metallb-operator-controller-manager   1/1     1            1           20m
deployment.apps/metallb-operator-webhook-server       1/1     1            1           20m

❯ kubectl get pod -n metallb-system -o wide
NAME                                                   READY   STATUS    RESTARTS   AGE   IP          NODE            NOMINATED NODE   READINESS GATES
metallb-operator-controller-manager-5dbc8fd577-bgczj   1/1     Running   0          21m   10.10.2.2   myk8s-worker3   <none>           <none>
metallb-operator-webhook-server-77d47cb764-9lcs8       1/1     Running   0          21m   10.10.3.3   myk8s-worker    <none>           <none>

❯ kubectl logs -n metallb-system -l app=metallb -f
Defaulted container "speaker" out of: speaker, frr, reloader, frr-metrics, cp-frr-files (init), cp-reloader (init), cp-metrics (init)
Defaulted container "speaker" out of: speaker, frr, reloader, frr-metrics, cp-frr-files (init), cp-reloader (init), cp-metrics (init)
Defaulted container "speaker" out of: speaker, frr, reloader, frr-metrics, cp-frr-files (init), cp-reloader (init), cp-metrics (init)
Defaulted container "speaker" out of: speaker, frr, reloader, frr-metrics, cp-frr-files (init), cp-reloader (init), cp-metrics (init)
error: you are attempting to follow 6 log streams, but maximum allowed concurrency is 5, use --max-log-requests to increase the limit

❯ cat << EOF | kubectl apply -f -
apiVersion: metallb.io/v1beta1
kind: MetalLB
metadata:
  name: metallb
  namespace: metallb-system
EOF
metallb.metallb.io/metallb created

❯ kubectl get pod -n metallb-system -o wide
NAME                                                   READY   STATUS             RESTARTS   AGE    IP           NODE                  NOMINATED NODE   READINESS GATES
controller-7dd49fb757-rsf9n                            0/1     ImagePullBackOff   0          118s   10.10.2.3    myk8s-worker3         <none>           <none>
metallb-operator-controller-manager-5dbc8fd577-bgczj   1/1     Running            0          36m    10.10.2.2    myk8s-worker3         <none>           <none>
metallb-operator-webhook-server-77d47cb764-9lcs8       1/1     Running            0          36m    10.10.3.3    myk8s-worker          <none>           <none>
speaker-ndwfb                                          0/4     Init:0/3           0          118s   172.18.0.3   myk8s-worker3         <none>           <none>
speaker-vnjlb                                          0/4     Init:0/3           0          118s   172.18.0.5   myk8s-worker          <none>           <none>
speaker-w9946                                          0/4     Init:0/3           0          118s   172.18.0.2   myk8s-worker2         <none>           <none>
speaker-zgf46                                          0/4     Init:0/3           0          118s   172.18.0.4   myk8s-control-plane   <none>           <none>

Events:
  Type     Reason     Age                  From               Message
  ----     ------     ----                 ----               -------
  Normal   Scheduled  2m30s                default-scheduler  Successfully assigned metallb-system/controller-7dd49fb757-rsf9n to myk8s-worker3
  Warning  Failed     57s                  kubelet            Failed to pull image "quay.io/metallb/controller:main": failed to pull and unpack image "quay.io/metallb/controller:main": failed to copy: read tcp 172.18.0.3:33674->104.18.37.147:443: read: connection reset by peer
  Warning  Failed     57s                  kubelet            Error: ErrImagePull
  Normal   BackOff    56s                  kubelet            Back-off pulling image "quay.io/metallb/controller:main"
  Warning  Failed     56s                  kubelet            Error: ImagePullBackOff
  Normal   Pulling    45s (x2 over 2m29s)  kubelet            Pulling image "quay.io/metallb/controller:main" 
  Normal   Pulled     31s                  kubelet            Successfully pulled image "quay.io/metallb/controller:main" in 13.488s (13.488s including waiting). Image size: 29150053 bytes.
  Normal   Created    31s                  kubelet            Created container controller
  Normal   Started    31s                  kubelet            Started container controller

❯ kubectl get pod -n metallb-system -o wide
NAME                                                   READY   STATUS    RESTARTS   AGE    IP           NODE                  NOMINATED NODE   READINESS GATES
controller-7dd49fb757-rsf9n                            1/1     Running   0          4m3s   10.10.2.3    myk8s-worker3         <none>           <none>
metallb-operator-controller-manager-5dbc8fd577-bgczj   1/1     Running   0          38m    10.10.2.2    myk8s-worker3         <none>           <none>
metallb-operator-webhook-server-77d47cb764-9lcs8       1/1     Running   0          38m    10.10.3.3    myk8s-worker          <none>           <none>
speaker-ndwfb                                          4/4     Running   0          4m3s   172.18.0.3   myk8s-worker3         <none>           <none>
speaker-vnjlb                                          3/4     Running   0          4m3s   172.18.0.5   myk8s-worker          <none>           <none>
speaker-w9946                                          3/4     Running   0          4m3s   172.18.0.2   myk8s-worker2         <none>           <none>
speaker-zgf46                                          4/4     Running   0          4m3s   172.18.0.4   myk8s-control-plane   <none>           <none>

❯ kubectl logs -n metallb-system -l app=metallb -f
Defaulted container "speaker" out of: speaker, frr, reloader, frr-metrics, cp-frr-files (init), cp-reloader (init), cp-metrics (init)
Defaulted container "speaker" out of: speaker, frr, reloader, frr-metrics, cp-frr-files (init), cp-reloader (init), cp-metrics (init)
Defaulted container "speaker" out of: speaker, frr, reloader, frr-metrics, cp-frr-files (init), cp-reloader (init), cp-metrics (init)
Defaulted container "speaker" out of: speaker, frr, reloader, frr-metrics, cp-frr-files (init), cp-reloader (init), cp-metrics (init)
error: you are attempting to follow 6 log streams, but maximum allowed concurrency is 5, use --max-log-requests to increase the limit

docker network ls
# (sol0) kind alias 대신 Id(SHA256)으로 확인해도 됩니다. 다 같은 조회방법일 뿐.  
docker network inspect kind
# (sol1) docker inspect kind
# (sol2) docker inspect <Id>
# (sol3) docker network inspect <Id>

docker ps -q | xargs docker inspect --format '{{.Name}} {{.NetworkSettings.Networks.kind.IPAddress}}'
# 위의 것 치기 귀찮으면?  
# 이건 각 Node에 할당 된 IP
# docker inspect e8 | grep IPv4Address
# 이건 브리지에 정의된 IP 대역
# docker inspect e8 | grep Subnet

❯ docker network ls
NETWORK ID     NAME      DRIVER    SCOPE
9a356b80a908   bridge    bridge    local
d2f5be011872   host      host      local
e8e5256f1aa7   kind      bridge    local
439c3626705a   none      null      local
❯ docker ps -q | xargs docker inspect --format '{{.Name}} {{.NetworkSettings.Networks.kind.IPAddress}}'
/myk8s-worker 172.18.0.2
/myk8s-control-plane 172.18.0.5
/myk8s-worker2 172.18.0.4
/myk8s-worker3 172.18.0.3
❯ docker inspect e8 | grep Subnet
                    "Subnet": "172.18.0.0/16",
                    "Subnet": "fc00:f853:ccd:e793::/64",
❯ docker inspect e8 | grep IPv4Address
                "IPv4Address": "172.18.0.3/16",
                "IPv4Address": "172.18.0.4/16",
                "IPv4Address": "172.18.0.2/16",
                "IPv4Address": "172.18.0.5/16",

kubectl explain ipaddresspools.metallb.io
kubectl explain l2advertisements.metallb.io

cat << EOF > metallb-ipaddresspool.yaml
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: kkumtree-ippool
  namespace: metallb-system
spec:
  addresses:
  - 172.18.255.200-172.18.255.254
EOF
kubectl apply -f metallb-ipaddresspool.yaml

cat << EOF > metallb-l2advertisement.yaml
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: kkumtree-l2adv
  namespace: metallb-system
spec:
  ipAddressPools:
  - kkumtree-ippool
EOF
kubectl apply -f metallb-l2advertisement.yaml

❯ kubectl get ipaddresspools,l2advertisements -n metallb-system
NAME                                       AUTO ASSIGN   AVOID BUGGY IPS   ADDRESSES
ipaddresspool.metallb.io/kkumtree-ippool   true          false             ["172.18.255.200-172.18.255.254"]

NAME                                        IPADDRESSPOOLS        IPADDRESSPOOL SELECTORS   INTERFACES
l2advertisement.metallb.io/kkumtree-l2adv   ["kkumtree-ippool"] 

cat <<EOF> metallb-test-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: svc1
spec:
  ports:
    - name: svc1-webport
      port: 80
      targetPort: 80
  selector:
    app: webpod
  type: LoadBalancer
---
apiVersion: v1
kind: Service
metadata:
  name: svc2
spec:
  ports:
    - name: svc2-webport
      port: 80
      targetPort: 80
  selector:
    app: webpod
  type: LoadBalancer
---
apiVersion: v1
kind: Service
metadata:
  name: svc3
spec:
  ports:
    - name: svc3-webport
      port: 80
      targetPort: 80
  selector:
    app: webpod
  type: LoadBalancer
EOF
kubectl apply -f metallb-test-svc.yaml

❯ docker exec -it myk8s-control-plane arp-scan --interfac=eth0 --localnet
Interface: eth0, type: EN10MB, MAC: 02:42:ac:12:00:05, IPv4: 172.18.0.5
Starting arp-scan 1.10.0 with 65536 hosts (https://github.com/royhills/arp-scan)
172.18.0.1	02:42:01:57:ec:6f	(Unknown: locally administered)
172.18.0.2	02:42:ac:12:00:02	(Unknown: locally administered)
172.18.0.3	02:42:ac:12:00:03	(Unknown: locally administered)
172.18.0.4	02:42:ac:12:00:04	(Unknown: locally administered)

❯ kubectl get svc,ep
NAME                 TYPE           CLUSTER-IP     EXTERNAL-IP      PORT(S)        AGE
service/kubernetes   ClusterIP      10.200.1.1     <none>           443/TCP        2d20h
service/svc1         LoadBalancer   10.200.1.66    172.18.255.200   80:31865/TCP   2m42s
service/svc2         LoadBalancer   10.200.1.133   172.18.255.201   80:30199/TCP   2m42s
service/svc3         LoadBalancer   10.200.1.175   172.18.255.202   80:30462/TCP   2m42s

NAME                   ENDPOINTS                   AGE
endpoints/kubernetes   172.18.0.5:6443             2d20h
endpoints/svc1         10.10.1.2:80,10.10.3.2:80   2m42s
endpoints/svc2         10.10.1.2:80,10.10.3.2:80   2m42s
endpoints/svc3         10.10.1.2:80,10.10.3.2:80   2m42s

❯ docker exec -it myk8s-control-plane arp-scan --interfac=eth0 --localnet
Interface: eth0, type: EN10MB, MAC: 02:42:ac:12:00:05, IPv4: 172.18.0.5
Starting arp-scan 1.10.0 with 65536 hosts (https://github.com/royhills/arp-scan)
172.18.0.1	02:42:01:57:ec:6f	(Unknown: locally administered)
172.18.0.2	02:42:ac:12:00:02	(Unknown: locally administered)
172.18.0.3	02:42:ac:12:00:03	(Unknown: locally administered)
172.18.0.4	02:42:ac:12:00:04	(Unknown: locally administered)
172.18.0.1	02:42:01:57:ec:6f	(Unknown: locally administered) (DUP: 2)
172.18.0.2	02:42:ac:12:00:02	(Unknown: locally administered) (DUP: 2)
172.18.0.3	02:42:ac:12:00:03	(Unknown: locally administered) (DUP: 2)
172.18.0.4	02:42:ac:12:00:04	(Unknown: locally administered) (DUP: 2)
172.18.0.1	02:42:01:57:ec:6f	(Unknown: locally administered) (DUP: 3)
172.18.255.200	02:42:ac:12:00:02	(Unknown: locally administered)
172.18.255.201	02:42:ac:12:00:02	(Unknown: locally administered)
172.18.255.202	02:42:ac:12:00:03	(Unknown: locally administered)
172.18.0.1	02:42:01:57:ec:6f	(Unknown: locally administered) (DUP: 4)
172.18.0.1	02:42:01:57:ec:6f	(Unknown: locally administered) (DUP: 5)

14 packets received by filter, 0 packets dropped by kernel
Ending arp-scan 1.10.0: 65536 hosts scanned in 263.158 seconds (249.04 hosts/sec). 7 responded

❯ kubectl describe svc | grep Events: -A5
Events:                   <none>


Name:                     svc1
Namespace:                default
Labels:                   <none>
--
Events:
  Type    Reason        Age   From                Message
  ----    ------        ----  ----                -------
  Normal  IPAllocated   11m   metallb-controller  Assigned IP ["172.18.255.200"]
  Normal  nodeAssigned  11m   metallb-speaker     announcing from node "myk8s-worker" with protocol "layer2"

--
Events:
  Type    Reason        Age                From                Message
  ----    ------        ----               ----                -------
  Normal  IPAllocated   11m                metallb-controller  Assigned IP ["172.18.255.201"]
  Normal  nodeAssigned  11m (x2 over 11m)  metallb-speaker     announcing from node "myk8s-worker" with protocol "layer2"

--
Events:
  Type    Reason        Age   From                Message
  ----    ------        ----  ----                -------
  Normal  IPAllocated   11m   metallb-controller  Assigned IP ["172.18.255.202"]
  Normal  nodeAssigned  11m   metallb-speaker     announcing from node "myk8s-worker3" with protocol "layer2"

❯ kubectl describe svc/svc2 | grep metallb-speaker
  Normal  nodeAssigned  13m (x2 over 13m)  metallb-speaker     announcing from node "myk8s-worker" with protocol "layer2"

cat <<EOT> 3pod.yaml
apiVersion: v1
kind: Pod
metadata:
  name: webpod1
  labels:
    app: webpod
spec:
  nodeName: myk8s-worker
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Pod
metadata:
  name: webpod2
  labels:
    app: webpod
spec:
  nodeName: myk8s-worker2
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Pod
metadata:
  name: webpod3
  labels:
    app: webpod
spec:
  nodeName: myk8s-worker3
  containers:
  - name: container
    image: traefik/whoami
  terminationGracePeriodSeconds: 0
EOT
kubectl apply -f 3pod.yaml

❯ kubectl get svc
NAME         TYPE           CLUSTER-IP     EXTERNAL-IP      PORT(S)        AGE
kubernetes   ClusterIP      10.200.1.1     <none>           443/TCP        3d3h
svc1         LoadBalancer   10.200.1.66    172.18.255.200   80:31865/TCP   6h52m
svc2         LoadBalancer   10.200.1.133   172.18.255.201   80:30199/TCP   6h52m
svc3         LoadBalancer   10.200.1.175   172.18.255.202   80:30462/TCP   6h52m
❯ curl -s 172.18.255.200
Hostname: webpod2
IP: 127.0.0.1
IP: ::1
IP: 10.10.1.2
IP: fe80::b862:52ff:fed1:7cbb
RemoteAddr: 172.18.0.2:3343
GET / HTTP/1.1
Host: 172.18.255.200
User-Agent: curl/8.5.0
Accept: */*

❯ curl -s 172.18.255.201
Hostname: webpod1
IP: 127.0.0.1
IP: ::1
IP: 10.10.3.2
IP: fe80::d0fa:f1ff:fe03:49bf
RemoteAddr: 10.10.3.1:20630
GET / HTTP/1.1
Host: 172.18.255.201
User-Agent: curl/8.5.0
Accept: */*

❯ curl -s 172.18.255.202
Hostname: webpod2
IP: 127.0.0.1
IP: ::1
IP: 10.10.1.2
IP: fe80::b862:52ff:fed1:7cbb
RemoteAddr: 172.18.0.3:6166
GET / HTTP/1.1
Host: 172.18.255.202
User-Agent: curl/8.5.0
Accept: */*

❯ docker -v
Docker version 24.0.7, build 24.0.7-0ubuntu4.1
❯ sudo docker network ls
NETWORK ID     NAME      DRIVER    SCOPE
a90c02431872   bridge    bridge    local
d2f5be011872   host      host      local
439c3626705a   none      null      local
❯ sudo docker network inspect bridge
[
    {
        "Name": "bridge",
        "Id": "a90c02431872f243e6c3918d0ca4f8875fb070ae0ad1a504891b74485634de14",
        "Created": "2024-10-02T08:22:04.247414071+09:00",
        "Scope": "local",
        "Driver": "bridge",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": null,
            "Config": [
                {
                    "Subnet": "172.17.0.0/16",
                    "Gateway": "172.17.0.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": false,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": {},
        "Options": {
            "com.docker.network.bridge.default_bridge": "true",
            "com.docker.network.bridge.enable_icc": "true",
            "com.docker.network.bridge.enable_ip_masquerade": "true",
            "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",
            "com.docker.network.bridge.name": "docker0",
            "com.docker.network.driver.mtu": "1500"
        },
        "Labels": {}
    }
]

ERROR: failed to create cluster: failed to list nodes: command "docker ps -a --filter label=io.x-k8s.kind.cluster=myk8s --format '{{.Names}}'" failed with error: exit status 1
Command Output: permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get "http://%2Fvar%2Frun%2Fdocker.sock/v1.24/containers/json?all=1&filters=%7B%22label%22%3A%7B%22io.x-k8s.kind.cluster%3Dmyk8s%22%3Atrue%7D%7D": dial unix /var/run/docker.sock: connect: permission denied

# https://snapcraft.io/docker refer and apply  
sudo addgroup --system docker
sudo adduser $USER docker
newgrp docker
sudo service docker restart

❯ sudo addgroup --system docker
info: The group `docker' already exists as a system group. Exiting.
❯ sudo adduser $USER docker
info: Adding user `kkumtree' to group `docker' ...
❯ newgrp docker
❯ sudo service docker restart

❯ kubectl get deployment -n metallb-system controller
NAME         READY   UP-TO-DATE   AVAILABLE   AGE
controller   1/1     1            1           2d17h

❯ kubectl get daemonset -n metallb-system speaker
NAME      DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR            AGE
speaker   4         4         4       4            4           kubernetes.io/os=linux   2d17h

❯ kubectl logs -n metallb-system -l app=metallb -f
error: you are attempting to follow 6 log streams, but maximum allowed concurrency is 5, use --max-log-requests to increase the limit

❯ kubectl logs -n metallb-system -l app=metallb -f --max-log-requests 6
# (중략) 로그는 잘 나옵니다.
failed to create fsnotify watcher: too many open files

# 1024인데 부족하다고?  
# 아 다른거도 켜고 하고 있었구나
❯ ulimit -n
1024

#<domain> <type> <item> <value>
* soft nofile 10000
* hard nofile 30000