maas-baremetal-k8s-tutorial/maas-setup.sh

238 lines
8.6 KiB
Bash
Raw Normal View History

2021-11-21 13:13:07 +00:00
# Copyright 2012-2021 Canonical Ltd. This software is licensed under the
2021-11-21 13:12:35 +00:00
# GNU Affero General Public License version 3 (see the file LICENSE).
# lxd / maas issue. either upgrade lxd or maas to 3.1
2021-11-17 10:56:08 +00:00
sudo snap switch --channel=latest/stable lxd
sudo snap refresh lxd
2021-11-16 16:41:11 +00:00
sudo snap install jq
2021-11-17 10:56:08 +00:00
sudo snap install --channel=3.1/edge maas
sudo snap install --channel=3.1/edge maas-test-db
2021-11-16 21:06:29 +00:00
# clone the git repository
2021-11-16 21:07:27 +00:00
cd ~
2021-11-16 21:06:29 +00:00
git clone https://github.com/antongisli/maas-baremetal-k8s-tutorial.git
# get local interface name (this assumes a single default route is present)
2021-10-27 10:57:40 +00:00
export INTERFACE=$(ip route | grep default | cut -d ' ' -f 5)
export IP_ADDRESS=$(ip -4 addr show dev $INTERFACE | grep -oP '(?<=inet\s)\d+(\.\d+){3}')
2021-11-16 16:02:39 +00:00
sudo sed -i 's/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/' /etc/sysctl.conf
sudo sysctl -p
sudo iptables -t nat -A POSTROUTING -o $INTERFACE -j SNAT --to $IP_ADDRESS
2021-10-27 10:57:40 +00:00
#TODO inbound port forwarding/load balancing
# Persist NAT configuration
echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections
echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections
2021-11-16 16:02:39 +00:00
sudo apt-get install iptables-persistent -y
2021-10-27 10:57:40 +00:00
# LXD init
2021-11-16 21:08:13 +00:00
sudo cat maas-baremetal-k8s-tutorial/lxd.conf | lxd init --preseed
2021-11-16 21:10:13 +00:00
# verify LXD network config
lxc network show lxdbr0
2021-10-27 10:57:40 +00:00
# Wait for LXD to be ready
lxd waitready
# Initialise MAAS
2021-11-16 16:02:39 +00:00
sudo maas init region+rack --database-uri maas-test-db:/// --maas-url http://${IP_ADDRESS}:5240/MAAS
2021-10-27 10:57:40 +00:00
sleep 15
# Create MAAS admin and grab API key
2021-11-16 21:11:38 +00:00
sudo maas createadmin --username admin --password admin --email admin
2021-11-16 16:02:39 +00:00
export APIKEY=$(sudo maas apikey --username admin)
2021-10-27 10:57:40 +00:00
# MAAS admin login
maas login admin 'http://localhost:5240/MAAS/' $APIKEY
# Configure MAAS networking (set gateways, vlans, DHCP on etc)
export SUBNET=10.10.10.0/24
export FABRIC_ID=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.fabric_id")
export VLAN_TAG=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.vid")
export PRIMARY_RACK=$(maas admin rack-controllers read | jq -r ".[] | .system_id")
maas admin subnet update $SUBNET gateway_ip=10.10.10.1
maas admin ipranges create type=dynamic start_ip=10.10.10.200 end_ip=10.10.10.254
maas admin vlan update $FABRIC_ID $VLAN_TAG dhcp_on=True primary_rack=$PRIMARY_RACK
maas admin maas set-config name=upstream_dns value=8.8.8.8
# Add LXD as a VM host for MAAS and capture the VM_HOST_ID
2021-11-17 10:56:08 +00:00
export VM_HOST_ID=$(maas admin vm-hosts create password=password type=lxd power_address=https://${IP_ADDRESS}:8443 \
2021-11-16 21:13:51 +00:00
project=maas | jq '.id')
2021-10-27 10:57:40 +00:00
2021-11-16 21:28:09 +00:00
# allow high CPU oversubscription so all VMs can use all cores
maas admin vm-host update $VM_HOST_ID cpu_over_commit_ratio=4
2021-11-17 08:36:55 +00:00
# create tags for MAAS
2021-11-17 10:56:08 +00:00
maas admin tags create name=juju-controller comment='This tag should to machines that will be used as juju controllers'
2021-11-17 08:36:55 +00:00
maas admin tags create name=metal comment='This tag should to machines that will be used as bare metal'
2021-11-03 17:08:33 +00:00
### creating VMs for Juju controller and our "bare metal"
2021-11-02 20:29:15 +00:00
# add a VM for the juju controller with minimal memory
2021-11-16 16:40:14 +00:00
maas admin vm-host compose $VM_HOST_ID cores=8 memory=2048 architecture="amd64/generic" \
storage="main:16(pool1)" hostname="juju-controller"
2021-11-03 17:08:33 +00:00
# get the system-id and tag the machine with "juju-controller"
2021-11-17 10:56:08 +00:00
export JUJU_SYSID=$(maas admin machines read | jq '.[]
2021-11-16 16:40:14 +00:00
| select(."hostname"=="juju-controller")
| .["system_id"]' | tr -d '"')
2021-11-03 17:08:33 +00:00
maas admin tag update-nodes "juju-controller" add=$JUJU_SYSID
2021-10-27 10:57:40 +00:00
2021-11-03 17:08:33 +00:00
## Create 3 "bare metal" machines and tag them with "metal"
for ID in 1 2 3
do
2021-11-16 16:40:14 +00:00
maas admin vm-host compose $VM_HOST_ID cores=8 memory=8192 architecture="amd64/generic" \
storage="main:25(pool1),ceph:100(pool1)" hostname="metal-${ID}"
2021-11-16 16:40:14 +00:00
SYSID=$(maas admin machines read | jq -r --arg MACHINE "metal-${ID}" '.[]
| select(."hostname"==$MACHINE)
| .["system_id"]' | tr -d '"')
maas admin tag update-nodes "metal" add=$SYSID
done
2021-11-03 13:13:00 +00:00
### Juju setup (note, this section requires manual intervention)
cd ~
2021-10-27 10:57:40 +00:00
sudo snap install juju --classic
2021-11-17 10:56:08 +00:00
sed -i "s/IP_ADDRESS/$IP_ADDRESS/" maas-baremetal-k8s-tutorial/maas-cloud.yaml
juju add-cloud --local maas-cloud maas-baremetal-k8s-tutorial/maas-cloud.yaml
2021-10-27 10:57:40 +00:00
juju add-credential maas-cloud
juju clouds --local
juju credentials
2021-11-02 20:29:15 +00:00
# Bootstrap the maas-cloud - get a coffee
juju bootstrap maas-cloud --bootstrap-constraints "tags=juju-controller mem=2G"
# fire up the juju gui to view the fun
2021-11-17 14:05:59 +00:00
# if it's a remote machine, you can use an SSH tunnel to get access to it:
# e.g. ssh ubuntu@x.x.x.x -L8080:10.10.10.2:17070
juju gui
# get coffee
2021-11-17 14:05:59 +00:00
# check jujus view of machines
juju machines
# add machines to juju from the maas cloud
# it will grab the 3 we already created since they are in a "READY state"
for ID in 1 2 3
do
juju add-machine
done
# take a look at machines list again, should see 3 machines
juju machines
2021-11-03 13:13:00 +00:00
### Ceph
2021-11-17 14:05:59 +00:00
# deploy ceph-mon to LXD VMs inside our metal machines
2021-11-02 20:29:15 +00:00
juju deploy -n 3 ceph-mon --to lxd:0,lxd:1,lxd:2
2021-11-17 14:05:59 +00:00
# deploy ceph-osd directly to the machines
juju deploy --config maas-baremetal-k8s-tutorial/ceph-osd.yaml cs:ceph-osd -n 3 --to 0,1,2
# relate ceph-mon and ceph-osd
juju add-relation ceph-mon ceph-osd
2021-11-03 13:13:00 +00:00
# watch the fun (with a another coffee).
2021-11-02 20:29:15 +00:00
watch -c juju status --color
2021-11-03 13:13:00 +00:00
# Wait for Ceph to settle before proceeding
2021-11-02 20:29:15 +00:00
2021-11-03 13:13:00 +00:00
### Kubernetes
# Deploy kubernetes-core with juju and re-use existing machines.
juju deploy kubernetes-core --map-machines=existing,0=0,1=1
2021-11-03 13:13:00 +00:00
# add the new kubernetes as a cloud to juju
mkdir ~/.kube
2021-11-17 20:08:00 +00:00
juju scp kubernetes-master/0:/home/ubuntu/config ~/.kube/config
# add storage relations
juju add-relation ceph-mon:admin kubernetes-master
juju add-relation ceph-mon:client kubernetes-master
2021-11-03 13:13:00 +00:00
# add k8s to juju (choose option 1, client only)
juju add-k8s my-k8s
2021-11-03 13:13:00 +00:00
juju bootstrap my-k8s
juju controllers
2021-11-03 13:13:00 +00:00
### Deploy a test application on K8s cluster
# Create a model in juju, which creates a namespace in K8s
juju add-model hello-kubecon
# Deploy the charm "hello-kubecon", and set a hostname for the ingress
juju deploy hello-kubecon --config juju-external-hostname=kubecon.test
# Deploy the ingress integrator - this is a helper to setup the ingress
juju deploy nginx-ingress-integrator ingress
2021-11-21 13:05:49 +00:00
# trust the ingress (it needs cluster credentials to make changes)
juju trust ingress --scope=cluster
# Relate our app to the ingress - this causes the ingress to be setup
juju relate hello-kubecon ingress
# Explore the setup
kubectl describe ingress -n hello-kubecon
kubectl get svc -n hello-kubecon
kubectl describe svc hello-kubecon-service -n hello-kubecon
kubectl get pods -n hello-kubecon
# Lastly, in order to be able to reach the service from outside our host machine,
# we can use port forwarding. Replace 10.10.10.5 with the IP seen on the ingress.
sudo iptables -t nat -A PREROUTING -p tcp -i enp6s0 \
--dport 8000 -j DNAT --to-destination 10.10.10.5:80
2021-11-20 16:45:05 +00:00
# if you want to persist this, run sudo dpkg-reconfigure iptables-persistent
2021-11-21 13:05:49 +00:00
# Now you should be able to open a browser and navigate to http://$IP_ADDRESS:8000
2021-11-20 16:45:05 +00:00
2021-11-21 13:05:49 +00:00
# scale our kubernetes cluster - find a machine
# Avoid kubernetes-master or existing kubernetes-worker machines
2021-11-20 18:34:30 +00:00
# https://discourse.charmhub.io/t/scaling-applications/1075
2021-11-20 16:45:05 +00:00
juju switch maas-cloud-default
juju status
# add a kubernetes-worker
juju add-unit kubernetes-worker --to 2
# add another kubecon unit
2021-11-20 19:08:50 +00:00
juju switch my-k8s
2021-11-20 16:45:05 +00:00
juju add-unit -n 1 hello-kubecon
juju status
# what happened to the ingress?
kubectl get ingress -n hello-kubecon
2021-11-21 13:05:49 +00:00
# exercise for the reader - iptables round robin or MetalLB:)
2021-11-20 16:45:05 +00:00
2021-11-20 18:34:30 +00:00
# scale down hello-kubecon
juju remove-unit --num-units 1 hello-kubecon
# scaledown kubernetes
juju switch maas-cloud-default
2021-11-21 13:05:49 +00:00
juju remove-unit kubernetes-worker/1
2021-11-20 18:34:30 +00:00
juju status
2021-11-20 16:45:05 +00:00
# if you want to test destroying your hello-kubecon:
2021-11-21 13:05:49 +00:00
juju switch my-k8s
2021-11-20 16:45:05 +00:00
juju destroy-model hello-kubecon --release-storage
# if you want to destroy your kubenetes controller for juju
2021-11-21 13:05:49 +00:00
juju switch maas-cloud-default
2021-11-20 16:45:05 +00:00
juju destroy-controller my-k8s
2021-11-21 13:05:49 +00:00
# if you want to remove your k8s cluster:
juju switch maas-cloud-default
juju remove-application kubernetes-master kubernetes-worker etcd flannel easyrsa
# if you want to remove ceph
juju switch maas-cloud-default
juju remove-application ceph-mon ceph-osd
# To clean up everything:
juju destroy-controller -y --destroy-all-models --destroy-storage maas-cloud-default
# And the machines created in MAAS can be deleted easily in the MAAS GUI.
2021-11-21 13:05:49 +00:00
### Reference materials notes
2021-11-03 13:13:00 +00:00
# https://jaas.ai/ceph-base
2021-11-16 12:47:15 +00:00
# https://jaas.ai/canonical-kubernetes/bundle/471
2021-11-18 17:38:43 +00:00
# https://medium.com/swlh/kubernetes-external-ip-service-type-5e5e9ad62fcd
# https://charmhub.io/nginx-ingress-integrator
# https://drive.google.com/file/d/1estQna40vz4uS5tBd9CvKdILdwAmcNFH/view - hello-kubecon
# https://ubuntu.com/kubernetes/docs/troubleshooting - troubleshooting
2021-11-21 13:05:49 +00:00
# https://juju.is/blog/deploying-mattermost-and-kubeflow-on-kubernetes-with-juju-2-9
### END
2021-11-18 17:38:43 +00:00