propellor

propellor config for hosts.
git clone git://git.ricketyspace.net/propellor.git
Log | Files | Refs | LICENSE

Sbuild.hs (15738B)


      1 {-# OPTIONS_HADDOCK prune #-}
      2 {-# LANGUAGE TypeFamilies #-}
      3 
      4 {-|
      5 Maintainer: Sean Whitton <spwhitton@spwhitton.name>
      6 
      7 Build and maintain schroots for use with sbuild.
      8 
      9 For convenience we set up several enhancements, such as ccache and eatmydata.
     10 This means we have to make several assumptions:
     11 
     12 1. you want to build for a Debian release strictly newer than squeeze, or for a
     13 Buntish release newer than or equal to trusty
     14 
     15 2. if you want to build for Debian stretch or newer, you have sbuild 0.70.0 or
     16 newer
     17 
     18 The latter is due to the migration from GnuPG v1 to GnuPG v2.1 in Debian
     19 stretch, which older sbuild can't handle.
     20 
     21 Suggested usage in @config.hs@:
     22 
     23 >  mybox = host "mybox.example.com" $ props
     24 >  	& osDebian (Stable "buster") X86_64
     25 >  	& Apt.useLocalCacher
     26 >  	& sidSchrootBuilt
     27 >  	& Sbuild.usableBy (User "spwhitton")
     28 >  	& Schroot.overlaysInTmpfs
     29 >    where
     30 >  	sidSchrootBuilt = Sbuild.built Sbuild.UseCcache $ props
     31 >  		& osDebian Unstable X86_32
     32 >  		& Sbuild.osDebianStandard
     33 >  		& Sbuild.update `period` Weekly (Just 1)
     34 >  		& Sbuild.useHostProxy mybox
     35 
     36 If you are using sbuild older than 0.70.0, you also need:
     37 
     38 >  & Sbuild.keypairGenerated
     39 
     40 To take advantage of the piuparts and autopkgtest support, add to your
     41 @~/.sbuildrc@ (assumes sbuild 0.71.0 or newer):
     42 
     43 >  $piuparts_opts = [
     44 >      '--no-eatmydata',
     45 >      '--schroot',
     46 >      '%r-%a-sbuild',
     47 >      '--fail-if-inadequate',
     48 >      ];
     49 >
     50 >  $autopkgtest_root_args = "";
     51 >  $autopkgtest_opts = ["--", "schroot", "%r-%a-sbuild"];
     52 
     53 On Debian jessie hosts, you should ensure that sbuild and autopkgtest come from
     54 the same suite.  This is because the autopkgtest binary changed its name between
     55 jessie and stretch.  If you have not installed backports of sbuild or
     56 autopkgtest, you don't need to do anything.  But if you have installed either
     57 package from jessie-backports (with Propellor or otherwise), you should install
     58 the other from jessie-backports, too.
     59 
     60 -}
     61 
     62 module Propellor.Property.Sbuild (
     63 	-- * Creating and updating sbuild schroots
     64 	UseCcache(..),
     65 	built,
     66 	-- * Properties for use inside sbuild schroots
     67 	update,
     68 	useHostProxy,
     69 	osDebianStandard,
     70 	-- * Global sbuild configuration
     71 	-- blockNetwork,
     72 	keypairGenerated,
     73 	keypairInsecurelyGenerated,
     74 	usableBy,
     75 	userConfig,
     76 ) where
     77 
     78 import Propellor.Base
     79 import Propellor.Types.Core
     80 import Propellor.Types.Info
     81 import Propellor.Property.Debootstrap (extractSuite)
     82 import qualified Propellor.Property.Apt as Apt
     83 import qualified Propellor.Property.Ccache as Ccache
     84 import qualified Propellor.Property.Chroot as Chroot
     85 import qualified Propellor.Property.ConfFile as ConfFile
     86 import qualified Propellor.Property.Debootstrap as Debootstrap
     87 import qualified Propellor.Property.File as File
     88 -- import qualified Propellor.Property.Firewall as Firewall
     89 import qualified Propellor.Property.Schroot as Schroot
     90 import qualified Propellor.Property.Reboot as Reboot
     91 import qualified Propellor.Property.Localdir as Localdir
     92 import qualified Propellor.Property.User as User
     93 
     94 import Data.List
     95 
     96 -- | Whether an sbuild schroot should use ccache during builds
     97 --
     98 -- ccache is generally useful but it breaks building some packages.  This data
     99 -- types allows you to toggle it on and off for particular schroots.
    100 data UseCcache = UseCcache | NoCcache
    101 
    102 -- | Build and configure a schroot for use with sbuild
    103 --
    104 -- The second parameter should specify, at a minimum, the operating system for
    105 -- the schroot.  This is usually done using a property like 'osDebian'
    106 built
    107 	:: UseCcache
    108 	-> Props metatypes
    109 	-> RevertableProperty (HasInfo + DebianLike) Linux
    110 built cc ps = case schrootSystem ps of
    111 	Nothing -> emitError
    112 	Just s@(System _ arch) -> case extractSuite s of
    113 		Nothing -> emitError
    114 		Just suite -> built' cc ps suite
    115 			(architectureToDebianArchString arch)
    116   where
    117 	schrootSystem :: Props metatypes -> Maybe System
    118 	schrootSystem (Props ps') = fromInfoVal . fromInfo $
    119 		mconcat (map getInfo ps')
    120 
    121 	emitError :: RevertableProperty (HasInfo + DebianLike) Linux
    122 	emitError = impossible theError <!> impossible theError
    123 	theError = "sbuild schroot does not specify suite and/or architecture"
    124 
    125 built'
    126 	:: UseCcache
    127 	-> Props metatypes
    128 	-> String
    129 	-> String
    130 	-> RevertableProperty (HasInfo + DebianLike) Linux
    131 built' cc (Props ps) suite arch = provisioned <!> deleted
    132   where
    133 	provisioned :: Property (HasInfo + DebianLike)
    134 	provisioned = combineProperties desc $ props
    135 		& cleanupOldConfig
    136 		& overlaysKernel
    137 		& preReqsInstalled
    138 		& ccacheMaybePrepared cc
    139 		& Chroot.provisioned schroot
    140 		& conf suite arch
    141 	  where
    142 		desc = "built sbuild schroot for " ++ suiteArch
    143 
    144 	-- TODO we should kill any sessions still using the chroot
    145 	-- before destroying it (as suggested by sbuild-destroychroot)
    146 	deleted :: Property Linux
    147 	deleted = combineProperties desc $ props
    148 		! Chroot.provisioned schroot
    149 		! compatSymlink
    150 		& File.notPresent schrootConf
    151 	  where
    152 		desc = "no sbuild schroot for " ++ suiteArch
    153 
    154 	conf suite' arch' = combineProperties "sbuild config file" $ props
    155 		& pair "description" (suite' ++ "/" ++ arch' ++ " autobuilder")
    156 		& pair "groups" "root,sbuild"
    157 		& pair "root-groups" "root,sbuild"
    158 		& pair "profile" "sbuild"
    159 		& pair "type" "directory"
    160 		& pair "directory" schrootRoot
    161 		& unionTypeOverlay
    162 		& aliasesLine
    163 		& pair "command-prefix" (intercalate "," commandPrefix)
    164 	  where
    165 		pair k v = ConfFile.containsIniSetting schrootConf
    166 			(suiteArch ++ "-sbuild", k, v)
    167 		unionTypeOverlay :: Property DebianLike
    168 		unionTypeOverlay = property' "add union-type = overlay" $ \w ->
    169 			Schroot.usesOverlays >>= \usesOverlays ->
    170 				if usesOverlays
    171 				then ensureProperty w $
    172 					pair "union-type" "overlay"
    173 				else noChange
    174 
    175 	compatSymlink = File.isSymlinkedTo
    176 		("/etc/sbuild/chroot" </> suiteArch ++ "-sbuild")
    177 		(File.LinkTarget schrootRoot)
    178 
    179 	-- if we're building a sid chroot, add useful aliases
    180 	-- In order to avoid more than one schroot getting the same aliases, we
    181 	-- only do this if the arch of the chroot equals the host arch.
    182 	aliasesLine :: Property UnixLike
    183 	aliasesLine = property' "maybe set aliases line" $ \w ->
    184 		sidHostArchSchroot suite arch >>= \isSidHostArchSchroot ->
    185 			if isSidHostArchSchroot
    186 			then ensureProperty w $
    187 				ConfFile.containsIniSetting schrootConf
    188 					( suiteArch ++ "-sbuild"
    189 					, "aliases"
    190 					, aliases
    191 					)
    192 			else return NoChange
    193 
    194 	-- if the user has indicated that this host should use
    195 	-- union-type=overlay schroots, we need to ensure that we have rebooted
    196 	-- to a kernel supporting OverlayFS.  Otherwise, executing sbuild(1)
    197 	-- will fail.
    198 	overlaysKernel :: Property DebianLike
    199 	overlaysKernel = property' "reboot for union-type=overlay" $ \w ->
    200 		Schroot.usesOverlays >>= \usesOverlays ->
    201 			if usesOverlays
    202 			then ensureProperty w $
    203 				Reboot.toKernelNewerThan "3.18"
    204 			else noChange
    205 
    206 	-- clean up config from earlier versions of this module
    207 	cleanupOldConfig :: Property UnixLike
    208 	cleanupOldConfig =
    209 		property' "old sbuild module config cleaned up" $ \w -> do
    210 			void $ ensureProperty w $
    211 				check (doesFileExist fstab)
    212 				(File.lacksLine fstab aptCacheLine)
    213 			void $ liftIO . tryIO $ removeDirectoryRecursive profile
    214 			void $ liftIO $ nukeFile schrootPiupartsConf
    215 			-- assume this did nothing
    216 			noChange
    217 	  where
    218 		fstab = "/etc/schroot/sbuild/fstab"
    219 		profile = "/etc/schroot/piuparts"
    220 		schrootPiupartsConf = "/etc/schroot/chroot.d"
    221 			</> suiteArch ++ "-piuparts-propellor"
    222 
    223 	-- the schroot itself
    224 	schroot = Chroot.debootstrapped Debootstrap.BuilddD
    225 			schrootRoot (Props schrootProps)
    226 	schrootProps =
    227 		ps ++ [toChildProperty $ Apt.installed ["eatmydata", "ccache"]
    228 		-- Drop /usr/local/propellor since build chroots should be
    229 		-- clean.  Note that propellor does not have to install its
    230 		-- build-deps into the chroot, so this is sufficient cleanup
    231 		, toChildProperty $ Localdir.removed]
    232 
    233 	-- static values
    234 	suiteArch = suite ++ "-" ++ arch
    235 	schrootRoot = "/srv/chroot" </> suiteArch
    236 	schrootConf = "/etc/schroot/chroot.d"
    237 		</> suiteArch ++ "-sbuild-propellor"
    238 	aliases = intercalate ","
    239 		[ "sid"
    240 		-- if the user wants to build for experimental, they would use
    241 		-- their sid chroot and sbuild's --extra-repository option to
    242 		-- enable experimental
    243 		, "rc-buggy"
    244 		, "experimental"
    245 		-- we assume that building for UNRELEASED means building for
    246 		-- unstable
    247 		, "UNRELEASED"
    248 		-- the following is for dgit compatibility:
    249 		, "UNRELEASED-"
    250 			++ arch
    251 			++ "-sbuild"
    252 		]
    253 	commandPrefix = case cc of
    254 		UseCcache -> "/var/cache/ccache-sbuild/sbuild-setup":base
    255 		_ -> base
    256 	  where
    257 		base = ["eatmydata"]
    258 
    259 -- | Properties that will be wanted in almost any Debian schroot, but not in
    260 -- schroots for other operating systems.
    261 osDebianStandard :: Property Debian
    262 osDebianStandard = propertyList "standard Debian sbuild properties" $ props
    263 	& Apt.stdSourcesList
    264 
    265 -- | Ensure that an sbuild schroot's packages and apt indexes are updated
    266 --
    267 -- This replaces use of sbuild-update(1).
    268 update :: Property DebianLike
    269 update = Apt.update `before` Apt.upgrade `before` Apt.autoRemove
    270 
    271 -- | Ensure that an sbuild schroot uses the host's Apt proxy.
    272 --
    273 -- This property is typically used when the host has 'Apt.useLocalCacher'.
    274 useHostProxy :: Host -> Property DebianLike
    275 useHostProxy h = property' "use host's apt proxy" $ \w ->
    276 	-- Note that we can't look at getProxyInfo outside the property,
    277 	-- as that would loop, but it's ok to look at it inside the
    278 	-- property. Thus the slightly strange construction here.
    279 	case getProxyInfo of
    280 		Just (Apt.HostAptProxy u) -> ensureProperty w (Apt.proxy' u)
    281 		Nothing -> noChange
    282   where
    283 	getProxyInfo = fromInfoVal . fromInfo . hostInfo $ h
    284 
    285 aptCacheLine :: String
    286 aptCacheLine = "/var/cache/apt/archives /var/cache/apt/archives none rw,bind 0 0"
    287 
    288 -- | Ensure that sbuild and associated utilities are installed
    289 preReqsInstalled :: Property DebianLike
    290 preReqsInstalled = Apt.installed ["piuparts", "autopkgtest", "lintian", "sbuild"]
    291 
    292 -- | Add an user to the sbuild group in order to use sbuild
    293 usableBy :: User -> Property DebianLike
    294 usableBy u = User.hasGroup u (Group "sbuild") `requires` preReqsInstalled
    295 
    296 -- | Generate the apt keys needed by sbuild
    297 --
    298 -- You only need this if you are using sbuild older than 0.70.0.
    299 keypairGenerated :: Property DebianLike
    300 keypairGenerated = check (not <$> doesFileExist secKeyFile) $ go
    301 	`requires` preReqsInstalled
    302 	-- Work around Debian bug #792100 which is present in Jessie.
    303 	-- Since this is a harmless mkdir, don't actually check the OS
    304 	`requires` File.dirExists "/root/.gnupg"
    305   where
    306 	go :: Property DebianLike
    307 	go = tightenTargets $
    308 		cmdProperty "sbuild-update" ["--keygen"]
    309 		`assume` MadeChange
    310 
    311 secKeyFile :: FilePath
    312 secKeyFile = "/var/lib/sbuild/apt-keys/sbuild-key.sec"
    313 
    314 -- | Generate the apt keys needed by sbuild using a low-quality source of
    315 -- randomness
    316 --
    317 -- Note that any running rngd will be killed; if you are using rngd, you should
    318 -- arrange for it to be restarted after this property has been ensured.  E.g.
    319 --
    320 -- >  & Sbuild.keypairInsecurelyGenerated
    321 -- >  	`onChange` Systemd.started "my-rngd-service"
    322 --
    323 -- Useful on throwaway build VMs.
    324 --
    325 -- You only need this if you are using sbuild older than 0.70.0.
    326 keypairInsecurelyGenerated :: Property DebianLike
    327 keypairInsecurelyGenerated = check (not <$> doesFileExist secKeyFile) go
    328   where
    329 	go :: Property DebianLike
    330 	go = combineProperties "sbuild keyring insecurely generated" $ props
    331 		& Apt.installed ["rng-tools"]
    332 		-- If this dir does not exist the sbuild key generation command
    333 		-- will fail; the user might have deleted it to work around
    334 		-- #831462
    335 		& File.dirExists "/var/lib/sbuild/apt-keys"
    336 		-- If there is already an rngd process running we have to kill
    337 		-- it, as it might not be feeding to /dev/urandom.  We can't
    338 		-- kill by pid file because that is not guaranteed to be the
    339 		-- default (/var/run/rngd.pid), so we killall
    340 		& userScriptProperty (User "root")
    341 			[ "start-stop-daemon -q -K -R 10 -o -n rngd"
    342 			, "rngd -r /dev/urandom"
    343 			]
    344 			`assume` MadeChange
    345 		& keypairGenerated
    346 		-- Kill off the rngd process we spawned
    347 		& userScriptProperty (User "root")
    348 			["kill $(cat /var/run/rngd.pid)"]
    349 			`assume` MadeChange
    350 
    351 ccacheMaybePrepared :: UseCcache -> Property DebianLike
    352 ccacheMaybePrepared cc = case cc of
    353 	UseCcache -> ccachePrepared
    354 	NoCcache  -> doNothing
    355 
    356 -- another script from wiki.d.o/sbuild
    357 ccachePrepared :: Property DebianLike
    358 ccachePrepared = propertyList "sbuild group ccache configured" $ props
    359 	-- We only set a limit on the cache if it doesn't already exist, so the
    360 	-- user can override our default limit
    361 	& check (not <$> doesDirectoryExist "/var/cache/ccache-sbuild")
    362 		(Ccache.hasLimits "/var/cache/ccache-sbuild" (Ccache.MaxSize "2G"))
    363 	`before` Ccache.hasCache (Group "sbuild") Ccache.NoLimit
    364 	& "/etc/schroot/sbuild/fstab" `File.containsLine`
    365 	"/var/cache/ccache-sbuild /var/cache/ccache-sbuild none rw,bind 0 0"
    366 		`describe` "ccache mounted in sbuild schroots"
    367 	& "/var/cache/ccache-sbuild/sbuild-setup" `File.hasContent`
    368 		[ "#!/bin/sh"
    369 		, ""
    370 		, "export CCACHE_DIR=/var/cache/ccache-sbuild"
    371 		, "export CCACHE_UMASK=002"
    372 		, "export CCACHE_COMPRESS=1"
    373 		, "unset CCACHE_HARDLINK"
    374 		, "export PATH=\"/usr/lib/ccache:$PATH\""
    375 		, ""
    376 		, "exec \"$@\""
    377 		]
    378 	& File.mode "/var/cache/ccache-sbuild/sbuild-setup"
    379 		(combineModes (readModes ++ executeModes))
    380 
    381 -- This doesn't seem to work with the current version of sbuild
    382 -- -- | Block network access during builds
    383 -- --
    384 -- -- This is a hack from <https://wiki.debian.org/sbuild> until #802850 and
    385 -- -- #802849 are resolved.
    386 -- blockNetwork :: Property Linux
    387 -- blockNetwork = Firewall.rule Firewall.OUTPUT Firewall.Filter Firewall.DROP
    388 -- 	(Firewall.GroupOwner (Group "sbuild")
    389 -- 	<> Firewall.NotDestination
    390 -- 		[Firewall.IPWithNumMask (IPv4 "127.0.0.1") 8])
    391 -- 	`requires` installed 	-- sbuild group must exist
    392 
    393 -- | Maintain recommended ~/.sbuildrc for a user, and adds them to the
    394 -- sbuild group
    395 --
    396 -- You probably want a custom ~/.sbuildrc on your workstation, but
    397 -- this property is handy for quickly setting up build boxes.
    398 --
    399 -- On Debian jessie hosts, you should ensure that sbuild and autopkgtest come
    400 -- from the same suite.  This is because the autopkgtest binary changed its name
    401 -- between jessie and stretch.  If you have not installed backports of sbuild or
    402 -- autopkgtest, you don't need to do anything.  But if you have installed either
    403 -- package from jessie-backports (with Propellor or otherwise), you should
    404 -- install the other from jessie-backports, too.
    405 userConfig :: User -> Property DebianLike
    406 userConfig user@(User u) = go
    407 	`requires` usableBy user
    408 	`requires` preReqsInstalled
    409   where
    410 	go :: Property DebianLike
    411 	go = property' ("~/.sbuildrc for " ++ u) $ \w -> do
    412 		h <- liftIO (User.homedir user)
    413 		ensureProperty w $ File.hasContent (h </> ".sbuildrc")
    414 			[ "$run_lintian = 1;"
    415 			, ""
    416 			, "$run_piuparts = 1;"
    417 			, "$piuparts_opts = ["
    418 			, "    '--no-eatmydata',"
    419 			, "    '--schroot',"
    420 			, "    '%r-%a-sbuild',"
    421 			, "    '--fail-if-inadequate',"
    422 			, "    ];"
    423 			, ""
    424 			, "$run_autopkgtest = 1;"
    425 			, "$autopkgtest_root_args = \"\";"
    426 			, "$autopkgtest_opts = [\"--\", \"schroot\", \"%r-%a-sbuild\"];"
    427 			]
    428 
    429 -- ==== utility functions ====
    430 
    431 -- Determine whether a schroot is
    432 --
    433 -- (i)  Debian sid, and
    434 -- (ii) the same architecture as the host.
    435 --
    436 -- This is the "sid host arch schroot".  It is considered the default schroot
    437 -- for sbuild builds, so we add useful aliases that work well with the suggested
    438 -- ~/.sbuildrc given in the haddock
    439 sidHostArchSchroot :: String -> String -> Propellor Bool
    440 sidHostArchSchroot suite arch = do
    441 	maybeOS <- getOS
    442 	return $ case maybeOS of
    443 		Nothing -> False
    444 		Just (System _ hostArch) ->
    445 			let hostArch' = architectureToDebianArchString hostArch
    446 			in suite == "unstable" && hostArch' == arch