Context Navigation

source: ccsd/trunk/crcnetd/_utils/ccsd_cfengine.py @ 1041

Last change on this file since 1041 was 1041, checked in by mglb1, 7 years ago
Ensure that the value passed to the propset command is always a string
Property svn:keywords set to `Id`
File size: 88.2 KB

Line
1	# Copyright (C) 2006 The University of Waikato
2	#
3	# This file is part of crcnetd - CRCnet Configuration System Daemon
4	#
5	# CFengine Configuration Setup
6	#
7	# Manages the cfengine configuration for the configuration system. The two
8	# primary tasks involved in this are:
9	# - Configuration file generation from templates
10	# - Controling cfengine runs and collecting output on demand
11	#
12	# Author: Matt Brown <matt@crc.net.nz>
13	# Version: $Id$
14	#
15	# crcnetd is free software; you can redistribute it and/or modify it under the
16	# terms of the GNU General Public License version 2 as published by the Free
17	# Software Foundation.
18	#
19	# crcnetd is distributed in the hope that it will be useful, but WITHOUT ANY
20	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
21	# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
22	# details.
23	#
24	# You should have received a copy of the GNU General Public License along with
25	# crcnetd; if not, write to the Free Software Foundation, Inc.,
26	# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
27	import sys
28	import os
29	import os.path
30	from stat import *
31	import svn.core as core
32	import svn.client as client
33	import svn.wc as wc
34	import svn.fs as fs
35	import svn.repos as repos
36	import imp
37	import time
38	import threading
39	from twisted.web import resource, server
40	from tempfile import mkdtemp
41	from Cheetah.Template import Template
42
43	from crcnetd._utils.ccsd_common import *
44	from crcnetd._utils.ccsd_log import *
45	from crcnetd._utils.ccsd_events import *
46	from crcnetd._utils.ccsd_config import config_get, config_get_required
47	from crcnetd._utils.ccsd_session import getSession, getSessionE
48	from crcnetd._utils.ccsd_server import registerResource, exportViaXMLRPC, \
49	initThread, registerRecurring, suggestThreadpoolSize
50
51	DEFAULT_MAX_THREADS = 10
52
53	KIND_DIR = core.svn_node_dir
54	KIND_FILE = core.svn_node_file
55	KIND_NONE = core.svn_node_none
56	KIND_UNKNOWN = core.svn_node_unknown
57	KIND_MAP = {KIND_DIR:"dir", KIND_FILE:"file", KIND_NONE:"none", \
58	KIND_UNKNOWN:"unknown"}
59
60	class ccs_template_error(ccsd_error):
61	pass
62	class ccs_cfengine_error(ccsd_error):
63	pass
64
65	#####################################################################
66	# General Cfengine Integration Helper Functions
67	#####################################################################
68	def initTemplates():
69	global _hostTemplates, _networkTemplates, _templateDir, _templateModDir
70
71	# Ensure output template module directories exist
72	ensureDirExists("%s/host" % _templateModDir)
73	ensureDirExists("%s/network" % _templateModDir)
74
75	# Ensure templates are compiled
76	rv = os.system("/usr/bin/cheetah compile -R --idir %s/host --odir " \
77	"%s/host --nobackup &>/dev/null" % (_templateDir, _templateModDir))
78	if rv != 0:
79	log_error("Unable to compile all host templates!")
80	rv = os.system("/usr/bin/cheetah compile -R --idir %s/network --odir " \
81	"%s/network --nobackup &>/dev/null" % \
82	(_templateDir, _templateModDir))
83	if rv != 0:
84	log_error("Unable to compile all network templates!")
85
86	# Load template files
87	_hostTemplates = loadTemplates("%s/host" % _templateModDir)
88	log_info("Loaded %s host templates for cfengine." % len(_hostTemplates))
89	_networkTemplates = loadTemplates("%s/network" % _templateModDir)
90	log_info("Loaded %s network templates for cfengine." % \
91	len(_networkTemplates))
92
93	# Register for events
94	registerEvents(_hostTemplates)
95	registerEvents(_networkTemplates)
96
97	def loadTemplates(moduleDir, baseStrip=""):
98	"""Creates a dictionary of template modules from the specified dir"""
99
100	# Dictionary of template objects
101	templates = {}
102
103	if baseStrip == "":
104	baseStrip = moduleDir
105
106	# Get a list of possible templates
107	if os.access(moduleDir, os.R_OK) != 1:
108	raise ccs_template_exception("Unable to access directory! - %s" % \
109	moduleDir)
110	ptempls = os.listdir(moduleDir)
111
112	# Scan through the list and load valid modules
113	for tfile in ptempls:
114	# Ignore hidden files
115	if tfile.startswith("."):
116	continue
117	tfilename = "%s/%s" % (moduleDir, tfile)
118	# Recurse into directories
119	if os.path.isdir(tfilename):
120	templates.update(loadTemplates(tfilename, baseStrip))
121	continue
122	# Ignore non module files
123	if not tfile.endswith(".py"):
124	continue
125	# Load the template module
126	tname = os.path.basename(tfilename)[:-3]
127	if tname == "__init__":
128	# Ignore python framework stuff
129	continue
130	m = None
131	try:
132	m = imp.load_source(tname, tfilename)
133	# Don't want the module in the system module list
134	if tname in sys.modules.keys():
135	del sys.modules[tname]
136	except:
137	log_debug("Module import failed for %s template" % tname, \
138	sys.exc_info())
139	pass
140	if not m:
141	log_error("Failed to import template: %s" % tname)
142	continue
143	# If an explicit output filename was specified use that, otherwise
144	# prepend path information and store the template filename.
145	path = "%s/" % os.path.dirname(tfilename[len(baseStrip)+1:])
146	if len(path) == 1:
147	path = ""
148	tclass = eval("m.%s" % tname)
149	m.fileName = "%s%s" % (path, getattr(tclass, "fileName", tname))
150	m.multiFile = getattr(tclass, "multiFile", False)
151	m.templateName = tname
152	# Store the template for future use, prepend path for uniqueness
153	templateID = "%s%s" % (path, tname)
154	if templateID in templates.keys():
155	log_error("Could not import duplicate template: %s" % tname)
156	continue
157	templates[templateID] = m
158
159	# Return the templates
160	return templates
161
162	def registerEvents(templates):
163	"""Runs through the list of templates and registers callbacks"""
164
165	for templateID, template in templates.items():
166	events = eval("template.%s.eventList" % template.templateName)
167	if events == [] or not events:
168	continue
169	for event in events:
170	registerCallback(templateID, event)
171
172	def registerCallback(templateID, eventName):
173	"""Keeps our internal list of what templates want to be called"""
174
175	if eventName in _events.keys():
176	_events[eventName].append(templateID)
177	return
178
179	try:
180	catchEvent(eventName, processEvent)
181	_events[eventName] = [templateID]
182	except ccs_event_error:
183	log_warn("Template %s not bound to event %s!" % \
184	(templateID, eventName), sys.exc_info())
185
186	def processEvent(eventName, host_id, session_id, **params):
187	"""Callback process from the event module.
188
189	This function is called when an event that a template has indicated
190	an interest in is fired. Look through our templates to find which
191	one is interested and call it.
192	"""
193	global _hostTemplates, _networkTemplates
194	from crcnetd.modules.ccs_host import getHostName
195
196	# Check session to see if a revision is active
197	session = getSessionE(session_id)
198	if session.revision == None:
199	return
200
201	host_name = getHostName(session_id, host_id)
202	hosts = {host_name:[],"network":[]}
203	# Fire all the templates attached to this event
204	for templateID in _events[eventName]:
205	# Process host templates if applicable
206	if host_id != -1 and templateID in _hostTemplates.keys():
207	hosts[host_name].append(templateID)
208	# Process any network templates
209	if templateID in _networkTemplates.keys():
210	hosts["network"].append(templateID)
211
212	# Fire off the generation job
213	generateTemplate(session_id, hosts)
214
215	@registerRecurring(60*10)
216	def cleanTemplateStatus():
217	"""Runs every ten minutes to clean up expired template status info"""
218	global _templateStats, _statsLock
219
220	# If generation is not finished by this many minutes after initiation
221	# there is an error, stats are removed in hope of causing a Key error
222	# that will kill the errant threads
223	err_time = 60*60
224	# Statistics for finished generation runs are removed this many minutes
225	# after the run completes
226	expire_time = 60*30
227
228	_statsLock.acquire()
229	try:
230	for key,stats in _templateStats.items():
231	if (time.time()-stats["initiated"]) > err_time:
232	# Still processing after one hour!?
233	log_error("Template generation (%s) was still active after " \
234	"%d seconds!" % (key, err_time))
235	del _templateStats[key]
236	continue
237	if stats["finished"] == 0:
238	# Still in progress
239	continue
240	if (time.time()-stats["finished"]) > expire_time:
241	log_info("Removing template generation status (%s)" % key)
242	del _templateStats[key]
243	finally:
244	_statsLock.release()
245
246	return True
247
248	@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR)
249	def getTemplateStatus(session_id, statsKey):
250	"""Returns the status of a specified template generation run"""
251	global _templateStats, _statsLock
252
253	stats = None
254
255	_statsLock.acquire()
256	try:
257	if statsKey not in _templateStats.keys():
258	raise ccs_cfengine_error("Invalid key: %s" % statsKey)
259	stats = dict(_templateStats[statsKey])
260	finally:
261	_statsLock.release()
262
263	return stats
264
265	@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
266	def generateTemplate(session_id, requested_templates={}):
267	"""Generates the specified configuration templates
268
269	The requested_templates parameter should be a dictionary indexed by
270	host_name each entry should contain a list of the template ids that
271	should be generated for that host.
272
273	If the template list for a host is empty all available templates will be
274	regenerated.
275	If the dictionary is completely empty then all hosts will have their
276	templates regenerated.
277
278	If an entry in the dictionary contains the key "network" it will be
279	interpreted as a list of network wide templates to regenerate. If this key
280	is absent then the network wide templates will be regenerated if the number
281	of host templates being regenerated is greater than one.
282
283	All template processing is shunted off to another thread and the user is
284	returned a token that can be passed to future calls to getTemplateStatus
285	to retrieve the current status of template generation. Generation results
286	are stored for 30 minutes after the end of template generation.
287	"""
288	global _hostTemplates, _networkTemplates, _templateStats, _statsLock
289	from crcnetd.modules.ccs_host import getHostList
290
291	# Setup the basic set of statistics about template generation
292	stats = {}
293	stats["planned"] = {}
294	stats["generated"] = {}
295	stats["initiated"] = time.time()
296	stats["setupprogress"] = 10
297	stats["generating"] = 0
298	stats["finished"] = 0
299	stats["error"] = None
300
301	# Work out what the user has requested us to generate
302	hosts = {}
303	host_names = requested_templates.keys()
304	if len(host_names) == 0:
305	host_names = [host["host_name"] for host in getHostList(session_id)]
306	for host in host_names:
307	if host == "network": continue
308	templates = []
309	if host in requested_templates.keys():
310	templates = requested_templates[host]
311	if len(templates) == 0:
312	templates = _hostTemplates.keys()
313	hosts[host] = templates
314	stats["planned"]["hosts"] = hosts
315	stats["generated"]["hosts"] = {}
316
317	# Work out how many host templates we're going to be generating
318	total = 0
319	for tlist in hosts.values():
320	total += len(tlist)
321	stats["planned"]["total"] = total
322
323	# Add network level templates if more than one host template is requested
324	# or they have been explicitly specified
325	if "network" in host_names:
326	stats["planned"]["network"] = requested_templates["network"]
327	elif total > 1:
328	stats["planned"]["network"] = _networkTemplates.keys()
329	else:
330	stats["planned"]["network"] = []
331	stats["planned"]["total"] += len(stats["planned"]["network"])
332	stats["generated"]["network"] = {}
333
334	# Aquire the lock to deal with statistics
335	statsKey = token = createPassword(8)
336	_statsLock.acquire()
337	try:
338	_templateStats[statsKey] = stats
339	finally:
340	_statsLock.release()
341
342	# Fire off the new thread
343	initThread(processTemplates, session_id, statsKey)
344
345	# Return back to the user
346	return statsKey
347
348	def processTemplates(session_id, statsKey):
349	"""Does the hardwork of generating templates in a thread"""
350	global _templateStats, _statsLock, _threadLimit
351	from crcnetd.modules.ccs_host import getHostList, ccs_host, \
352	getDistributions
353	from crcnetd._utils.ccsd_service import getServiceTemplateVars
354	from crcnetd.modules.ccs_asset import getAssetTypeTemplateVariables
355
356	log_info("Entered template processing thread (%s)" % statsKey)
357
358	# Make sure we have a revision to insert files into
359	# Check session to see if a revision is active
360	session = getSessionE(session_id)
361	commit=0
362	if session.revision == None:
363	session.begin("Updated host configuration files", initiator="cfengine")
364	commit=1
365	outputDir = session.revision.getConfigBase()
366
367	# More progress
368	_statsLock.acquire()
369	try:
370	_templateStats[statsKey]["setupProgress"] = 15
371	finally:
372	_statsLock.release()
373
374	# Load all the data that we need to pass to the templates
375	variables = {}
376	hlist = {}
377	id2name = {}
378	for host in getHostList(session_id):
379	host = ccs_host(session_id, host["host_id"])
380	hostDetails = host.getTemplateVariables()
381	hlist[host["host_name"]] = hostDetails
382	_statsLock.acquire()
383	try:
384	if _templateStats[statsKey]["setupProgress"] < 70:
385	_templateStats[statsKey]["setupProgress"] += 2
386	finally:
387	_statsLock.release()
388	variables["hosts"] = hlist
389	dlist = {}
390	for distrib in getDistributions(session_id):
391	dlist[distrib["distribution_id"]] = distrib
392	variables["distributions"] = dlist
393	variables["services"] = getServiceTemplateVars(session_id)
394	_statsLock.acquire()
395	try:
396	_templateStats[statsKey]["setupProgress"] = 80
397	finally:
398	_statsLock.release()
399	variables["date"] = time.ctime()
400	variables["domain"] = config_get_required("network", "domain")
401	variables["site_name"] = config_get_required("network", "site_name")
402	variables["server_name"] = config_get_required("network", "server_name")
403	variables["policy_ip"] = getIP(variables["server_name"])
404	variables["smtp_server"] = config_get_required("network", "smtp_server")
405	variables["admin_email"] = config_get_required("network", "admin_email")
406	_statsLock.acquire()
407	try:
408	_templateStats[statsKey]["setupProgress"] = 90
409	finally:
410	_statsLock.release()
411
412	variables["asset_types"] = getAssetTypeTemplateVariables(session_id)
413	variables["session_id"] = session_id
414
415	# Update the stats and get the list of hosts
416	_statsLock.acquire()
417	try:
418	planned = _templateStats[statsKey]["planned"]
419	_templateStats[statsKey]["generating"] = time.time()
420	_templateStats[statsKey]["setupProgress"] = 100
421	finally:
422	# Release the lock before proceeding
423	_statsLock.release()
424
425	# Now generate each hosts template in a separate thread
426	for host,template_ids in planned["hosts"].items():
427	# See if we can start a new thread
428	_threadLimit.acquire()
429	try:
430	# Extract the host specific data for this template
431	hostData = variables["hosts"][host]
432	# Fire off the thread to process this host, the thread will release
433	# the semaphore as it is about to exit
434	# XXX: This relies on the thread behaving nicely, need to find a
435	# way to find out if it's been bad and exited without releasing
436	# the semaphore
437	initThread(processHostTemplates, statsKey, outputDir, hostData, \
438	variables)
439	except:
440	log_error("Failed to start thread to process templates for " \
441	"host: %s!" % host, sys.exc_info())
442	_threadLimit.release()
443
444	# Wait until all the configuration files have been generated
445	hosts = list(planned["hosts"].keys())
446	hasError = False
447	hasTemplateError = False
448	while len(hosts) > 0:
449	# Wait a bit before trying again
450	time.sleep(1)
451	_statsLock.acquire()
452	try:
453	generated = _templateStats[statsKey]["generated"]["hosts"]
454	for host in hosts:
455	if not host in generated.keys():
456	continue
457	# Check if it finished successfully
458	if generated[host]["finished"] > 0:
459	# Remove from list
460	hosts.remove(host)
461	# Check if any of the templates failed to be output
462	if generated[host]["error"] == "template":
463	# Set the flag so we don't commit the revision
464	hasTemplateError = True
465	else:
466	# Check for fatal errors
467	if generated[host]["error"] != "":
468	# Set the flag and remove the host from processing
469	hosts.remove(host)
470	hasError = True
471	continue
472	finally:
473	_statsLock.release()
474
475	# Hosts are done, network templates now
476	for template_id in planned["network"]:
477	# See if we can start a new thread
478	_threadLimit.acquire()
479	try:
480	# Fire off the thread to process this template, the thread will
481	# release the semaphore as it is about to exit
482	# XXX: This relies on the thread behaving nicely, need to find a
483	# way to find out if it's been bad and exited without releasing
484	# the semaphore
485	initThread(processNetworkTemplate, statsKey, outputDir, \
486	template_id, variables)
487	except:
488	log_error("Failed to start thread to process network template: " \
489	"%s!" % template_id, sys.exc_info())
490	_threadLimit.release()
491
492	# Wait until all network templates are done
493	network = list(planned["network"])
494	while len(network) > 0:
495	# Wait a bit before trying again
496	time.sleep(1)
497	_statsLock.acquire()
498	try:
499	generated = _templateStats[statsKey]["generated"]["network"]
500	for template in network:
501	if not template in generated.keys():
502	continue
503	# Check for errors
504	if generated[template]["error"] != "":
505	hasTemplateError = True
506	# Remove from the list
507	network.remove(template)
508	finally:
509	_statsLock.release()
510
511	# Commit if neceesary and no template/host errors occured
512	rv = {"revision":None}
513	if commit and not (hasError or hasTemplateError):
514	rv = session.commit()
515
516	# All done
517	_statsLock.acquire()
518	try:
519	_templateStats[statsKey]["finished"] = time.time()
520	_templateStats[statsKey]["revision"] = rv["revision"]
521	d = _templateStats[statsKey]["finished"] - \
522	_templateStats[statsKey]["initiated"]
523	if hasError:
524	_templateStats[statsKey]["error"] = "host"
525	elif hasTemplateError:
526	_templateStats[statsKey]["error"] = "template"
527	finally:
528	_statsLock.release()
529
530	log_info("Template generation completed in %0.3f seconds for %s" % \
531	(d, statsKey))
532	return True
533
534	def processHostTemplates(statsKey, outputDir, hostData, networkData):
535	"""Runs in a thread and generates the output files for the template"""
536	global _templateStats, _statsLock, _threadLimit, _hostTemplates
537	host_id = -1
538	host = "unknown"
539
540	# It is imperative that we release the semaphore before returning from
541	# this function or we could potentially starve the calling thread of
542	# workers and we'd end up in a deadlock situation!
543	try:
544	# Work out the host_id and host_name of this host
545	host_id = hostData["host_id"]
546	host = hostData["host_name"]
547	hostPath = "%s/hosts/%s" % (outputDir, host)
548	start = time.time()
549	log_debug("Started host generation thread for %s" % host)
550	session = getSessionE(networkData["session_id"])
551
552	# Retrieve the list of templates to generate and flag the start
553	_statsLock.acquire()
554	try:
555	planned = _templateStats[statsKey]["planned"]
556	generated = _templateStats[statsKey]["generated"]["hosts"]
557	generated[host] = {}
558	generated[host]["initiated"] = start
559	generated[host]["finished"] = 0
560	generated[host]["error"] = ""
561	finally:
562	_statsLock.release()
563
564	# Now that we have a set of templates to use do the actual generation
565	hadError = False
566	for tname in planned["hosts"][host]:
567	try:
568	tstart = time.time()
569	# Instantiate a template
570	template = _hostTemplates[tname]
571	t = eval("template.%s()" % template.templateName)
572	# Is it enabled on this host?
573	if not t.enabledOnHost(networkData["session_id"], host_id):
574	continue
575	# Set up the variables we want to substitute in
576	t._searchList = [hostData, networkData]
577	# Generate the file
578	filename = "%s/%s" % (hostPath, template.fileName)
579	files = t.writeTemplate(filename, template.multiFile)
580	tend = time.time()
581	# Set properties
582	format = getattr(t, "highlightFormat", "")
583	if format != "":
584	for file in files:
585	session.revision.propset(file, "ccs:format", format)
586	# Record how long the file took to generate
587	_statsLock.acquire()
588	try:
589	info = {"time":tend-tstart, "error":""}
590	generated[host][tname] = info
591	finally:
592	_statsLock.release()
593	except:
594	(type, value, tb) = sys.exc_info()
595	# Single template errors can be skipped over, nothing will
596	# get committed unless the user explicitly approves
597	_statsLock.acquire()
598	try:
599
600	info = {"time":0, "error":value}
601	generated[host][tname] = info
602	finally:
603	_statsLock.release()
604	hadError = True
605	log_error("Failed to process template (%s) for host: %s" % \
606	(tname, host), (type, value, tb))
607
608	# Clean up
609	end = time.time()
610	log_debug("Completed host generation for %s in %0.3f seconds" % \
611	(host, (end-start)))
612	_statsLock.acquire()
613	try:
614	generated[host]["finished"] = end
615	# If a template failed make a note of it here
616	if hadError:
617	generated[host]["error"] = "template"
618	finally:
619	_statsLock.release()
620	except:
621	(type, value, tb) = sys.exc_info()
622	# Record the error for display to the user
623	_statsLock.acquire()
624	try:
625	generated = _templateStats[statsKey]["generated"]["hosts"]
626	if host not in generated.keys():
627	generated[host] = {}
628	generated[host]["initiated"] = 0
629	generated[host]["finished"] = time.time()
630	generated[host]["error"] = value
631	finally:
632	_statsLock.release()
633	# Log the error for the administrators record
634	log_error("Exception while processing host templates (%s)!" % \
635	host, (type, value, tb))
636
637	# Release the semaphore
638	_threadLimit.release()
639	return
640
641	def processNetworkTemplate(statsKey, outputDir, template_id, networkData):
642	"""Runs in a thread and generates the output files for the template"""
643	global _templateStats, _statsLock, _threadLimit, _networkTemplates
644
645	# It is imperative that we release the semaphore before returning from
646	# this function or we could potentially starve the calling thread of
647	# workers and we'd end up in a deadlock situation!
648	try:
649	log_debug("Started template generation thread for %s" % template_id)
650	tstart = time.time()
651	session = getSessionE(networkData["session_id"])
652	# Instantiate a template
653	template = _networkTemplates[template_id]
654	t = eval("template.%s()" % template.templateName)
655	# Set up the variables we want to substitute in
656	t._searchList = [networkData]
657	# Generate the file
658	filename = "%s/%s" % (outputDir, template.fileName)
659	files = t.writeTemplate(filename, template.multiFile)
660	tend = time.time()
661	# Set properties
662	format = getattr(template, "highlightFormat", "")
663	if format != "":
664	for file in files:
665	session.revision.propset(file, "ccs:format", format)
666	# Record how long the file took to generate
667	_statsLock.acquire()
668	try:
669	generated = _templateStats[statsKey]["generated"]["network"]
670	info = {"time":tend-tstart, "error":""}
671	generated[template_id] = info
672	finally:
673	_statsLock.release()
674	log_debug("Completed template generation for %s in %0.3f seconds" \
675	% (template_id, (tend-tstart)))
676	except:
677	(type, value, tb) = sys.exc_info()
678	# Skip over the error, nothing will get committed unless the user
679	# explicitly approves
680	_statsLock.acquire()
681	try:
682	generated = _templateStats[statsKey]["generated"]["network"]
683	info = {"time":0, "error":value}
684	generated[template_id] = info
685	finally:
686	_statsLock.release()
687	log_error("Failed to process network template (%s)" % template_id, \
688	(type, value, tb))
689
690	# Release the semaphore
691	_threadLimit.release()
692	return
693
694	#####################################################################
695	# Template Mixin
696	#####################################################################
697	class ccs_template(Template):
698	"""Config System Template Processor
699
700	All templates that expect to be processed by the configuration system
701	must derive from this class. It provides helper functions that are required
702	to determine where and when each template should be generated and how to
703	store it to disk.
704	"""
705
706	def __init__(self):
707	"""Initialise the class
708
709	You must call this method from your subclass
710	"""
711	# Call Cheetah's init
712	Template.__init__(self)
713
714	def writeTemplate(self, filename, multiFile):
715	"""Writes the template output to the specified file"""
716
717	files = []
718
719	# Ensure the output directory exists
720	ensureDirExists(os.path.dirname(filename))
721
722	# Get the template contents
723	template = self.writeBody().strip()
724	if not template.endswith("\n"): template += "\n"
725
726	# Write it out to a file
727	if not multiFile:
728	f = open(filename, "w")
729	f.write(template)
730	f.close()
731	files.append(filename)
732	else:
733	# Handle templates that generate multiple output files
734	lines = template.split("\n")
735	f = None
736	for line in lines:
737	if line.startswith(".newfile"):
738	# New file starting, close previous file
739	if f is not None: f.close()
740	# Open new file
741	parts = line.split(" ")
742	if len(parts) != 2:
743	raise ccs_template_error("Invalid multifile template!")
744	fname = "%s%s" % (filename, parts[1])
745	f = open(fname, "w")
746	files.append(fname)
747	continue
748	elif f is not None:
749	# Write the line out
750	f.write("%s\n" % line)
751	# Close the file
752	if f is not None: f.close()
753
754	return files
755
756	def __str__(self):
757	return self.writeBody()
758
759	def enabledOnHost(self, session_id, host_id):
760	"""Returns true if the template is applicable to the specified host
761
762	By default this method looks to see if the template has defined a
763	serviceName parameter. If it has then the function checks to see if
764	that service is enabled on the specified host. If it is not the
765	function returns False.
766
767	This function may be overriden by other classes/templates if you want
768	to implement more logic than the default implementation provides.
769	"""
770	serviceName = getattr(self, "serviceName", None)
771	if serviceName is None:
772	return True
773
774	from crcnetd.modules.ccs_host import ccs_host
775	host = ccs_host(session_id, host_id)
776	return host.hasServiceEnabledByName(serviceName)
777
778	def getTemplateVariables(self):
779	"""Returns a dictionary of variables that can be used by the template
780
781	The dictionary is passed to Cheetah's searchList so that it's entries
782	can be used as placeholders in the template.
783
784	This function retains an empty list. You should override this in your
785	implementing class.
786	"""
787	return []
788
789	#####################################################################
790	# Functions to maintain / deal with hosts
791	#####################################################################
792	def getCfengineHostStatus(session_id, host):
793	"""Returns a dictionary describing the state of the host configuration"""
794
795	status = {}
796
797	# Check keys exist
798	revision = ccs_revision(checkout=False)
799
800	sshkeydir = "inputs/sshkeys/%s" % host["host_name"]
801	cfkeydir = "ppkeys"
802
803	# SSH keys
804	status["ssh_key_status"] = STATUS_OK
805	status["ssh_key_text"] = ""
806	for key in ["dsa", "rsa"]:
807	if not revision.fileExists("%s/ssh_host_%s_key" % (sshkeydir, key)):
808	status["ssh_key_status"] = STATUS_CRITICAL
809	status["ssh_key_text"] += "%s private key missing, " % \
810	key.upper()
811	if not revision.fileExists("%s/ssh_host_%s_key.pub" % \
812	(sshkeydir, key)):
813	status["ssh_key_status"] = STATUS_CRITICAL
814	status["ssh_key_text"] += "%s private key missing, " % \
815	key.upper()
816	if status["ssh_key_text"]!="":
817	status["ssh_key_text"] = status["ssh_key_text"][:-2]
818
819	# Cfengine keys
820	status["cfengine_key_status"] = STATUS_OK
821	status["cfengine_key_text"] = ""
822	if not revision.fileExists("%s/root-%s.priv" % \
823	(cfkeydir, host["ip_address"])):
824	status["cfengine_key_status"] = STATUS_CRITICAL
825	status["cfengine_key_text"] += "CFengine private key missing"
826	if not revision.fileExists("%s/root-%s.pub" % \
827	(cfkeydir, host["ip_address"])):
828	status["cfengine_key_status"] = STATUS_CRITICAL
829	if status["cfengine_key_text"]!="": status["cfengine_key_text"]+=", "
830	status["cfengine_key_text"] += "CFengine public key missing"
831
832	return status
833
834	@catchEvent("hostAdded")
835	def hostAddedCB(eventName, host_id, session_id, **params):
836	"""Callback function to setup host configuration upon host creation"""
837
838	session = getSession(session_id)
839	if session is None:
840	return
841
842	# Create keys for the host
843	createSSHHostKeys(session_id, host_id)
844	createCfengineHostKeys(session_id, host_id)
845
846	@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
847	def createSSHHostKeys(session_id, host_id):
848	"""Creates a set of SSH keys for the specified host"""
849	from crcnetd.modules.ccs_host import ccs_host
850	session = getSessionE(session_id)
851
852	host = ccs_host(session_id, host_id)
853
854	# Check for existing changeset
855	commit = 0
856	if session.changeset == 0:
857	session.begin("Created SSH keys for %s" % host["host_name"])
858	commit = 1
859
860	try:
861	# Determine directories to store things in
862	revdir = session.revision.getConfigBase()
863	sshkeydir = "%s/sshkeys/%s" % (revdir, host["host_name"])
864	ensureDirExists(sshkeydir)
865
866	# Generate the keys
867	genSSHKey(sshkeydir, "rsa")
868	genSSHKey(sshkeydir, "dsa")
869	except:
870	(type, value, tb) = sys.exc_info()
871	log_error("Failed to generate SSH keys for %s! - %s" % \
872	(host["host_name"], value))
873	# Rollback any changes
874	if commit == 1:
875	session.rollback()
876	return False
877
878	# Commit changeset if necessary
879	if commit==1:
880	session.commit()
881
882	return True
883
884	@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
885	def createCfengineHostKeys(session_id, host_id):
886	"""Creates a set of Cfengine keys for the specified host"""
887	from crcnetd.modules.ccs_host import ccs_host
888	session = getSessionE(session_id)
889
890	host = ccs_host(session_id, host_id)
891
892	# Check for existing changeset
893	commit = 0
894	if session.changeset == 0:
895	session.begin("Created CFengine keys for %s" % host["host_name"])
896	commit = 1
897
898	try:
899	# Determine directories to store things in
900	workdir = session.revision.getWorkingDir()
901	cfkeydir = "%s/ppkeys" % (workdir)
902	ensureDirExists(cfkeydir)
903
904	# Generate the keys
905	genCfKey(cfkeydir, host["ip_address"])
906	except:
907	(type, value, tb) = sys.exc_info()
908	log_error("Failed to generate CFengine keys for %s! - %s" % \
909	(host["host_name"], value))
910	# Rollback any changes
911	if commit == 1:
912	session.rollback()
913	return False
914
915	# Commit changeset if necessary
916	if commit==1:
917	session.commit()
918
919	return True
920
921	def genSSHKey(dir, type):
922	"""Generates a SSH host key of the specified type
923
924	dir specifies the directory to place the key in
925	"""
926	filename = "ssh_host_%s_key" % type
927	key = "%s/%s" % (dir, filename)
928	pubkey = "%s.pub" % key
929
930	# Try and avoid needless recreating keys
931	if os.path.exists(key) and os.path.exists(pubkey):
932	log_debug("Skipping SSH key creation in %s. " \
933	"Keys already exist!" % dir)
934	return
935	try:
936	os.remove(key)
937	os.remove(pubkey)
938	except:
939	pass
940
941	fh = os.popen("cd %s &>/dev/null && /usr/bin/ssh-keygen -q -f %s " \
942	"-N '' -t %s \|\| echo \"unable to access %s\"" % \
943	(dir, filename, type, dir))
944	output = fh.readlines()
945	rv = fh.close()
946	if rv != None or (len(output)>0 and output[0].startswith("unable")):
947	raise ccs_cfengine_error("Could not create SSH %s host key: %s" % \
948	(type, "".join(output)))
949
950	def genCfKey(dir, host_ip):
951	"""Generates a Cfengine host key pair
952
953	dir specifies the directory to place the key in
954	"""
955	filename = "root-%s" % host_ip
956	key = "%s/%s.priv" % (dir, filename)
957	pubkey = "%s/%s.pub" % (dir, filename)
958
959	# Try and avoid needless recreating keys
960	if os.path.exists(key) and os.path.exists(pubkey):
961	log_debug("Skipping Cfengine key creation in %s. " \
962	"Keys already exist!" % dir)
963	return
964	try:
965	os.remove(key)
966	os.remove(pubkey)
967	except:
968	pass
969
970	fh = os.popen("cd %s &>/dev/null && /usr/sbin/cfkey -f %s \|\| echo " \
971	"\"unable to access %s\"" % (dir, filename, dir))
972	output = fh.readlines()
973	rv = fh.close()
974	if rv != None or (len(output)>0 and output[0].startswith("unable")):
975	raise ccs_cfengine_error("Could not create CFengine host key: %s" % \
976	"".join(output))
977
978	def getActiveRevision(hostname=None):
979	"""Returns the active revision for the cfengine input directory
980
981	If a hostname is specified it looks in the hosts/<hostname> directory
982	specifically
983	"""
984	global _cfInputDir
985
986	if hostname is None:
987	dir = _cfInputDir
988	else:
989	dir = "%s/hosts/%s" % (_cfInputDir, hostname)
990
991	# Get the revision number of the checked out revision
992	try:
993	contents = open("%s/ccs-revision" % dir, "r").read()
994	return int(contents.strip())
995	except:
996	log_debug("Could not read ccs-revision", sys.exc_info())
997	pass
998
999	# No checked out revision
1000	return -1
1001
1002	def getGeneratedRevision(hostname=None):
1003	"""Returns the latest revision that has been generated
1004
1005	If a hostname is specified it looks in the hosts/<hostname> directory
1006	specifically
1007	"""
1008
1009	if hostname is None:
1010	dir = ""
1011	else:
1012	dir = "inputs/hosts/%s" % hostname
1013
1014	revision = ccs_revision(checkout=False)
1015	return revision.getYoungestRevision(dir)
1016
1017	@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR)
1018	def getCfRunLogs(session_id):
1019	"""Returns the output of the last cfrun execution for all active hosts"""
1020	from crcnetd.modules.ccs_host import getHostList, getHostID
1021	from crcnetd.modules.ccs_status import getHostStatus, ccs_status_error, \
1022	ccs_host_status
1023	global _cfBaseDir, _cfrunStatus
1024
1025	hosts = {}
1026
1027	outputDir = "%s/outputs" % _cfBaseDir
1028	domain = config_get_required("network", "domain")
1029
1030	# Get a list of active hosts
1031	for host in getHostList(session_id):
1032	# Skip hosts that are disabled
1033	if not host["host_active"]:
1034	continue
1035	# Store enabled hosts
1036	hostname = host["host_name"]
1037	hosts[hostname] = host
1038	# Retrieve the information about it
1039	filename = "%s/%s.%s" % (outputDir, host["host_name"], domain)
1040	hosts[hostname]["last_run"] = -1
1041	hosts[hostname]["age"] = ""
1042	hosts[hostname]["age_rounded"] = ""
1043	hosts[hostname]["contents"] = []
1044	hosts[hostname]["no_lines"] = 0
1045	try:
1046	hostStatus = getHostStatus(hostname)
1047	except ccs_status_error:
1048	# Create a blank status entry
1049	host_id = getHostID(ADMIN_SESSION_ID, hostname)
1050	hostStatus = ccs_host_status(ADMIN_SESSION_ID, host_id)
1051	hosts[hostname]["active_rev"] = hostStatus.activeRevision
1052	hosts[hostname]["generated_rev"] = hostStatus.generatedRevision
1053	hosts[hostname]["operating_rev_status"] = \
1054	hostStatus.operatingRevisionStatus
1055	hosts[hostname]["operating_rev_text"] = \
1056	hostStatus.operatingRevisionText
1057	hosts[hostname]["operating_rev_hint"] = \
1058	hostStatus.operatingRevisionHint
1059	if hostStatus.infoUpdatedAt > 0:
1060	hosts[hostname]["operating_rev"] = int(hostStatus.operatingRevision)
1061	try:
1062	o = int(hosts[hostname]["operating_rev"])
1063	a = int(hosts[hostname]["active_rev"])
1064	g = int(hosts[hostname]["generated_rev"])
1065	if o > g:
1066	hosts[hostname]["operating_rev_status"] = "critical"
1067	hosts[hostname]["operating_rev_text"] = "Invalid Revision"
1068	hosts[hostname]["operating_rev_hint"] = "The operating " \
1069	"revision is greater than the newest generated " \
1070	"revision!"
1071	elif o != a:
1072	hosts[hostname]["operating_rev_status"] = "warning"
1073	hosts[hostname]["operating_rev_text"] = ""
1074	hosts[hostname]["operating_rev_hint"] = "The operating " \
1075	"revision does not match the desired active " \
1076	"revision!"
1077	except: pass
1078	else:
1079	hosts[hostname]["operating_rev"] = ""
1080	if hostname in _cfrunStatus.keys():
1081	hosts[hostname]["run_status"] = _cfrunStatus[hostname]["status"]
1082	hosts[hostname]["run_age"] = time.strftime("%d/%m/%Y %H:%M:%S", \
1083	time.localtime(_cfrunStatus[hostname]["time"]))
1084	hosts[hostname]["run_age_rounded"] = roundTime(time.time() - \
1085	_cfrunStatus[hostname]["time"])
1086	else:
1087	hosts[hostname]["run_status"] = ""
1088	hosts[hostname]["run_age"] = ""
1089	hosts[hostname]["run_age_rounded"] = ""
1090	hosts[hostname]["error"] = ""
1091	if not os.path.exists(filename):
1092	# No cfrun output for this host
1093	continue
1094	try:
1095	stat = os.stat("%s/%s.%s" % (outputDir, host["host_name"], domain))
1096	hosts[hostname]["age"] = time.strftime("%d/%m/%Y %H:%M:%S", \
1097	time.localtime(stat[ST_MTIME]))
1098	hosts[hostname]["age_rounded"] = roundTime(time.time() - \
1099	stat[ST_MTIME])
1100	hosts[hostname]["last_run"] = stat[ST_MTIME]
1101	hosts[hostname]["contents"] = open(filename).read()
1102	hosts[hostname]["no_lines"] = \
1103	len(hosts[hostname]["contents"].split("\n"))
1104	except:
1105	(type, value, tb) = sys.exc_info()
1106	log_error("Failed to retrieve CfRun log for %s" % hostname, \
1107	(type, value, tb))
1108	hosts[hostname]["error"] = value
1109	continue
1110
1111	# Return it
1112	return hosts
1113
1114	@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
1115	def pushConfig(session_id, hosts, params=[]):
1116	"""Initiates a cfrun instance to push a new config out to a host"""
1117	global _threadLimit, _cfrunStatus, _cfrunLock
1118
1119	# PHP sends in an associative array we want the values only
1120	hosts = hosts.values()
1121
1122	# Loop through the specified hosts and mark them as pending
1123	_cfrunLock.acquire()
1124	try:
1125	try:
1126	for host in hosts:
1127	_cfrunStatus[host] = {"status":"pending","time":time.time()}
1128	except:
1129	log_error("Exception setting up config push!", sys.exc_info())
1130	raise ccs_cfengine_error("Could not initiate config push!")
1131	finally:
1132	_cfrunLock.release()
1133
1134	# Start the threads
1135	initThread(startCfRunThreads, hosts, params)
1136	return True
1137
1138	def startCfRunThreads(hosts, params):
1139
1140	# Loop through the hosts and start threads to do the cfrun stuff
1141	for host in hosts:
1142	_threadLimit.acquire()
1143	try:
1144	initThread(doCfrun, host, params)
1145	except:
1146	log_error("Failed to start cfrun instance for host: %s!" % \
1147	host, sys.exc_info())
1148	_threadLimit.release()
1149	return True
1150
1151	def doCfrun(host_name, params=[]):
1152	"""Executes a cfrun instance to update the specified host
1153
1154	This function expects to be called in a thread and will take care of
1155	releasing the semaphore when it finishes.
1156	"""
1157	global _threadLimit, _cfrunStatus, _cfrunLock, _cfBaseDir
1158
1159	outputDir = "%s/outputs" % _cfBaseDir
1160	domain = config_get_required("network", "domain")
1161	cmdline = ""
1162	filename = ""
1163
1164	# Make sure that any errors are captured so that we can't exit without
1165	# releasing the semaphore
1166	try:
1167	# Record that this host is being processed
1168	_cfrunLock.acquire()
1169	try:
1170	if host_name in _cfrunStatus.keys():
1171	if _cfrunStatus[host_name]["status"] != "pending":
1172	# This host is already being processed!
1173	_threadLimit.release()
1174	return
1175	_cfrunStatus[host_name] = {"status":"running","time":time.time()}
1176	finally:
1177	_cfrunLock.release()
1178	# Ouput filename
1179	filename = "%s/%s.%s" % (outputDir, host_name, domain)
1180	try:
1181	os.unlink(filename)
1182	except:
1183	log_warn("Could not remove old logfile before cfrun! - %s" % \
1184	filename, sys.exc_info())
1185	# Build the command line
1186	cmdline = "/usr/sbin/cfrun %s" % host_name
1187	if len(params) > 0:
1188	cmdline += " -- "
1189	for i,param in params.items(): cmdline += "%s " % param
1190	# Execute the command
1191	fp = os.popen(cmdline, "r")
1192	output = fp.read().strip()
1193	rv = fp.close()
1194	if rv is not None:
1195	code="failed"
1196	else:
1197	code="succeeded"
1198	# Retrieve the actual cfrun output
1199	try:
1200	contents = open(filename, "r").read()
1201	except:
1202	log_warn("Could not read logfile from cfrun! - %s" % filename, \
1203	sys.exc_info())
1204	contents = ""
1205	# Prepend the output from the execution to the log
1206	fp = open(filename, "w")
1207	fp.write("%s\n" % (""80))
1208	fp.write("Command execution %s!\n" % code)
1209	fp.write("Cmdline: %s\n" % cmdline)
1210	if len(output) > 0:
1211	fp.write("\n%s\n" % output)
1212	fp.write("%s\n\n" % (""80))
1213	fp.write(contents)
1214	fp.close()
1215	# Record that this host is no longer being processed
1216	_cfrunLock.acquire()
1217	try:
1218	del _cfrunStatus[host_name]
1219	finally:
1220	_cfrunLock.release()
1221	except:
1222	(type, value, tb) = sys.exc_info()
1223	# Record that this host is no longer being processed
1224	_cfrunLock.acquire()
1225	try:
1226	del _cfrunStatus[host_name]
1227	finally:
1228	_cfrunLock.release()
1229	# Record an error in the log file
1230	if filename != "":
1231	try:
1232	try:
1233	contents = open(filename, "r").read()
1234	except:
1235	log_warn("Could not read logfile from cfrun! - %s" % \
1236	filename, sys.exc_info())
1237	contents = ""
1238	fp = open(filename, "w")
1239	fp.write("%s\n" % (""80))
1240	fp.write("Command execution failed!\n")
1241	fp.write("Cmdline: %s\n" % cmdline)
1242	fp.write("Error: %s\n" % value)
1243	fp.write("%s\n\n" % (""80))
1244	fp.write(contents)
1245	fp.close()
1246	except: pass
1247	log_error("Failed to initiate cfrun for host: %s" % host_name, \
1248	(type, value, tb))
1249
1250	# Release the semaphore
1251	_threadLimit.release()
1252	return
1253
1254	#####################################################################
1255	# Functions to maintain / deal with checked out Cfengine configs
1256	#####################################################################
1257	@catchEvent("revisionCreated")
1258	def updateCfInputDirCB(eventName, host_id, session_id, **kwargs):
1259	"""Callback function to trigger updates when revisions are created"""
1260
1261	autoupdate = int(config_get("cfengine", "autoupdate", "0"))
1262	if autoupdate:
1263	updateCfInputDir(session_id)
1264
1265	@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
1266	def updateCfInputDir(session_id, revNum=-1):
1267	"""Updates the cfengine input directory.
1268
1269	Triggered explicitly by user or on the revisionCreated hook.
1270	"""
1271	global _svnroot, _cfInputDir
1272
1273	# Get a python memory pool and context to use
1274	pool = core.svn_pool_create(None)
1275	ctx = client.svn_client_ctx_t()
1276	ctx.config = core.svn_config_get_config(None, pool)
1277
1278	# Work out which revision to update to
1279	rev = core.svn_opt_revision_t()
1280	if revNum == -1:
1281	rev.kind = core.svn_opt_revision_head
1282	else:
1283	rev.kind = core.svn_opt_revision_number
1284	rev.value.number = int(revNum)
1285
1286	# Checkout the current configuration HEAD to this directory
1287	rev = core.svn_opt_revision_t()
1288
1289	# See if there is a checked out revision
1290	gotrev = 0
1291	try:
1292	list = client.svn_client_ls(_cfInputDir, rev, False, ctx, pool)
1293	gotrev = 1
1294	except:
1295	# No checked out revision, remove bogus directory
1296	removeDir(_cfInputDir)
1297	ensureDirExists(_cfInputDir)
1298	pass
1299
1300	if gotrev:
1301	# Update the directory
1302	nrev = client.svn_client_update(_cfInputDir, rev, True, ctx, pool)
1303	else:
1304	# Checkout the directory
1305	nrev = client.svn_client_checkout(ccs_revision.svnroot, \
1306	_cfInputDir, rev, True, ctx, pool)
1307
1308	# Get rid of the pool
1309	core.svn_pool_destroy(pool)
1310	del pool
1311	del ctx
1312
1313	# Update the revision information files
1314	log_command("%s/update-revisioninfo \"%s\"" % (_cfInputDir, _cfInputDir))
1315
1316	log_info("Updated cfengine configuration to revision %s." % nrev)
1317
1318	#####################################################################
1319	# Functions to support the web based configuration browser
1320	#####################################################################
1321	@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR)
1322	def viewConfigAtPath(session_id, path, revno=None, order=None, direction=0, \
1323	doMarkup=True):
1324	"""Returns the configurations present at the specified path in the repo
1325
1326	If the path is a directory you get a directory listing back
1327	If the path is a file you get the file contents back along with other
1328	metadata.
1329
1330	If the revno parameter is set the configuration is shown as it existed
1331	in the specified revision. If the revno parameter is None then the
1332	current (HEAD) revision is shown.
1333
1334	The order parameter may be used to specify a parameter and order to sort
1335	the list that is returned. Valid parameters are name, size and date. Order
1336	may be 1 for descending or 0 for ascending.
1337
1338	Unless the markup parameter is set to false the contents of the file will
1339	be processed through enscript and HTML will be returned.
1340	"""
1341
1342	items = {}
1343
1344	revision = ccs_revision(checkout=False)
1345	path = path.replace("//","/")
1346
1347	(revno, entries) = revision.ls(path, revno)
1348	if revno == -1:
1349	# An error occured
1350	raise ccs_cfengine_error("Could not retrieve config at path: %s" % \
1351	path)
1352	for entry in entries:
1353	entry["path"] = entry["name"]
1354	entry["name"] = os.path.basename(entry["name"])
1355	if entry["path"].find("%s/" % entry["name"]) != -1:
1356	# psyvn bug, filename is duplicated
1357	entry["path"] = entry["path"][:-(len(entry["name"])+1)]
1358	if entry["path"].startswith(revision.svnroot):
1359	entry["path"] = entry["path"][len(revision.svnroot)+1:]
1360	entry["age"] = time.strftime("%d/%m/%Y %H:%M:%S", \
1361	time.localtime(entry["time"]))
1362	entry["age_rounded"] = roundTime(time.time() - entry["time"])
1363	entry["kind_desc"] = KIND_MAP[entry["kind"]]
1364
1365	# Sort if necessary
1366	if order is not None and order != "":
1367	direction = int(direction)
1368	if order not in ["name", "size", "date"]:
1369	raise ccs_cfengine_error("Invalid order parameter: %s" % order)
1370	if direction != 0 and direction != 1:
1371	raise ccs_cfengine_error("Invalid direction parameter: %s" % \
1372	direction)
1373	if order == "date":
1374	order="time"
1375	direction += 1
1376	if direction == 1:
1377	entries.sort(lambda x,y: cmp(x[order], y[order])*-1)
1378	else:
1379	entries.sort(lambda x,y: cmp(x[order], y[order]))
1380
1381	# Return now if there are multiple files or no files
1382	if len(entries) == 0 or len(entries) > 1:
1383	return [revno, entries]
1384	# or a single directory
1385	if entries[0]["kind"] == KIND_DIR:
1386	return [revno, entries]
1387
1388	# Single file, does it match the path we were given?
1389	if entries[0]["path"] == path:
1390	# Get the contents
1391	file = revision.getFile(path)
1392	props = revision.getProps(path)
1393	entries[0]["props"] = props
1394	if doMarkup:
1395	if "ccs:format" in props.keys():
1396	entries[0]["contents"] = markup(file, props["ccs:format"])
1397	else:
1398	entries[0]["contents"] = markup(file)
1399
1400	return [revno, entries]
1401
1402	def markup(file, format="sh"):
1403	"""Passes the file through enscript for syntax highlighting"""
1404
1405	# Generate the command line to pass to enscript
1406	cmdline = config_get("cfengine", "enscript_path", "enscript")
1407	cmdline += ' --color -h -q --language=html -p - -E' + format
1408	log_debug("Enscript command line: %s" % cmdline)
1409
1410	# Run enscript
1411	(fdi, fdo) = os.popen2(cmdline)
1412	fdi.write(file)
1413	fdi.close()
1414	odata = fdo.read()
1415	rv = fdo.close()
1416	if rv is not None:
1417	log_warn("Could not run enscript to markup file!")
1418	log_debug(odata)
1419	return file.split("\n")
1420
1421	# Strip header and footer
1422	i = odata.find('<PRE>')
1423	beg = i > 0 and i + 6
1424	i = odata.rfind('</PRE>')
1425	end = i > 0 and i or len(odata)
1426	odata = EnscriptDeuglifier().format(odata[beg:end])
1427	return odata.splitlines()
1428
1429	# Enscript Deuglifier code from Trac
1430	#
1431	# Copyright (C) 2003-2006 Edgewall Software
1432	# All rights reserved.
1433	#
1434	# Licensed using the Modified BSD license
1435	class EnscriptDeuglifier(object):
1436	def __new__(cls):
1437	self = object.__new__(cls)
1438	if not hasattr(cls, '_compiled_rules'):
1439	cls._compiled_rules = re.compile('(?:'+'\|'.join(cls.rules())+')')
1440	self._compiled_rules = cls._compiled_rules
1441	return self
1442
1443	def format(self, indata):
1444	return re.sub(self._compiled_rules, self.replace, indata)
1445
1446	def replace(self, fullmatch):
1447	for mtype, match in fullmatch.groupdict().items():
1448	if match:
1449	if mtype == 'font':
1450	return '<span>'
1451	elif mtype == 'endfont':
1452	return '</span>'
1453	return '<span class="code-%s">' % mtype
1454
1455	def rules(cls):
1456	return [
1457	r'(?P<comment><FONT COLOR="#B22222">)',
1458	r'(?P<keyword><FONT COLOR="#5F9EA0">)',
1459	r'(?P<type><FONT COLOR="#228B22">)',
1460	r'(?P<string><FONT COLOR="#BC8F8F">)',
1461	r'(?P<func><FONT COLOR="#0000FF">)',
1462	r'(?P<prep><FONT COLOR="#B8860B">)',
1463	r'(?P<lang><FONT COLOR="#A020F0">)',
1464	r'(?P<var><FONT COLOR="#DA70D6">)',
1465	r'(?P<font><FONT.*?>)',
1466	r'(?P<endfont></FONT>)'
1467	]
1468	rules = classmethod(rules)
1469
1470	#####################################################################
1471	# Cfengine Resource Server
1472	#####################################################################
1473	class cfserver(resource.Resource):
1474	"""Implements a simple webserver to retrieve resources related to cfengine
1475
1476	Called when the main server receives a query for /cfengine/*
1477
1478	Currently supports the following resources
1479	/cfengine/update.conf/<host_name>
1480	Retrieves an update.conf file for the specified host
1481	/cfkey/keyname
1482	Retrieves the specified cfengine key
1483
1484	Requests to this resource are restricted to the IP address ranges
1485	specified in the allow_requests_from configuration directive.
1486	"""
1487
1488	resourceName = "CFengine Resource Server"
1489
1490	# Mark as leaf so that render gets called
1491	isLeaf = 1
1492
1493	def render(self, request):
1494
1495	# XXX: Verify request IP
1496
1497	log_debug("Received cfengine request for %s" % request.path)
1498
1499	parts = request.path.split("/")
1500	path = "/".join(parts[2:])
1501
1502	# Process request
1503	if path.startswith("update.conf"):
1504	return self.getUpdateConf(path, request)
1505	elif path.startswith("cfkey"):
1506	return self.getCfkey(path, request)
1507	elif path.startswith("hostvars"):
1508	return self.getHostvars(path, request)
1509	else:
1510	request.setResponseCode(404, "Unknown cfengine resource requested!")
1511	request.finish()
1512	return server.NOT_DONE_YET
1513
1514	def getCfkey(self, path, request):
1515	"""Returns the contents of a cfengine key for the specified host."""
1516
1517	parts = path.split("/")
1518
1519	# Get a revision to grab the key from
1520	revision = ccs_revision()
1521	keydir = "%s/ppkeys" % revision.getWorkingDir()
1522
1523	# Retrieve the requested key
1524	keyfile = "%s/%s" % (keydir, parts[1])
1525
1526	try:
1527	fd = open(keyfile, "r")
1528	content = fd.readlines()
1529	fd.close()
1530	except:
1531	log_warn("Could not open Cfengine key file!", sys.exc_info())
1532	request.setResponseCode(404, "Unknown cfengine key requested!")
1533	request.finish()
1534	return server.NOT_DONE_YET
1535
1536	return "".join(content)
1537
1538	def getUpdateConf(self, path, request):
1539	"""Returns the contents of an update.conf file for the specified host.
1540
1541	Currently the host parameter is ignored.
1542	"""
1543
1544	# Get a revision to grab the update.conf from
1545	revision = ccs_revision()
1546	revdir = revision.getConfigBase()
1547
1548	try:
1549	fd = open("%s/cfconf/update.conf" % revdir, "r")
1550	except:
1551	request.setResponseCode(503, "update.conf unavailable. Please " \
1552	"try again later")
1553	request.finish()
1554	return server.NOT_DONE_YET
1555
1556	contents = fd.readlines()
1557	fd.close()
1558
1559	return "".join(contents)
1560
1561	def getHostvars(self, path, request):
1562	"""Returns the contents of an update.conf file for the specified host.
1563
1564	Currently the host parameter is ignored.
1565	"""
1566
1567	parts = path.split("/")
1568	if len(parts)<2:
1569	request.setResponseCode(404, "Hostname not specified" % host)
1570	request.finish()
1571	return server.NOT_DONE_YET
1572	host = parts[1]
1573
1574	# Get a revision to grab the hostvars file from
1575	revision = ccs_revision()
1576	revdir = revision.getConfigBase()
1577
1578	try:
1579	fd = open("%s/hosts/%s/cf.hostvars" % (revdir, host), "r")
1580	except:
1581	request.setResponseCode(404, "%s hostvars not found!" % host)
1582	request.finish()
1583	return server.NOT_DONE_YET
1584
1585	contents = fd.readlines()
1586	fd.close()
1587
1588	return "".join(contents)
1589
1590	#####################################################################
1591	# Revision Class
1592	#####################################################################
1593	class ccs_revision:
1594	"""Wrapper for a revision of files.
1595
1596	This class wraps the generation of an entire revision of the configuration
1597	files managed by this system.
1598
1599	It contains the methods needed to generate the files, insert them into
1600	version control (svn) and then update then pass the resulting revision
1601	identifier back to it's caller.
1602	"""
1603
1604	# Location of the svnroot
1605	svnroot = None
1606
1607	def __init__(self, parentSession=None, changeset=None, checkout=True):
1608	"""Creates a ccs_revision class.
1609
1610	If parentSession or changeset are not specified or None, a read-only
1611	revision is created. This is only useful if you want to inspect the
1612	repository without making any changes.
1613	"""
1614
1615	self.mParentSession = parentSession
1616	self.mChangeset = changeset
1617	self.checkout = checkout
1618	self.pendingProps = {}
1619	self.lock = threading.RLock()
1620	self._revinfocache = {}
1621	self._needsCommit = False
1622
1623	# Get a python memory pool and context to use
1624	self.pool = core.svn_pool_create(None)
1625	self.ctx = client.svn_client_ctx_t()
1626	self.ctx.config = core.svn_config_get_config(None, self.pool)
1627
1628	if self.checkout:
1629	# Setup a working directory for this revision
1630	self.rDir = mkdtemp("", "ccsd")
1631
1632	# Checkout the current configuration HEAD to this directory
1633	rev = core.svn_opt_revision_t()
1634	rev.kind = core.svn_opt_revision_head
1635	client.svn_client_checkout(self.svnroot, self.rDir, rev, \
1636	True, self.ctx, self.pool)
1637	self.mCurRev = rev
1638
1639	# Check basic repository structure
1640	if self.mParentSession is not None and self.mChangeset is not None:
1641	self.checkRepoStructure()
1642	else:
1643	self.rDir = None
1644	self.mCurRev = None
1645
1646	# Start with no errors
1647	self.mErrors = {}
1648
1649	def __del__(self):
1650
1651	# Nothing to check if nothing was checked out
1652	if self.rDir is None:
1653	core.svn_pool_destroy(self.pool)
1654	self.pool = None
1655	self.ctx = None
1656	return
1657
1658	# Don't check the status of a read-only revision
1659	if self.mParentSession is None or self.mChangeset is None:
1660	removeDir(self.rDir)
1661	core.svn_pool_destroy(self.pool)
1662	self.pool = None
1663	self.ctx = None
1664	return
1665
1666	# Callback function to process status information
1667	def status_cb(epath, entry):
1668	try:
1669	if entry.text_status == wc.svn_wc_status_ignored:
1670	pass
1671	elif entry.text_status != wc.svn_wc_status_normal:
1672	log_debug("Changes (%s) to %s in changeset %s will " \
1673	"be lost" % \
1674	(entry.text_status, epath, self.mChangeset))
1675	except:
1676	log_error("Failed to check status of %s" % epath, \
1677	sys.exc_info())
1678
1679	# Check the status
1680	self.flag = False
1681	self.lock.acquire()
1682	try:
1683	client.svn_client_status(self.rDir, self.mCurRev, status_cb, \
1684	True, True, False, False, self.ctx, self.pool)
1685	finally:
1686	self.lock.release()
1687
1688	# Clean up the working directory
1689	removeDir(self.rDir)
1690	core.svn_pool_destroy(self.pool)
1691	self.pool = None
1692	self.ctx = None
1693
1694	def getWorkingDir(self):
1695	"""Returns the path that the repository is checked out into"""
1696
1697	return self.rDir
1698
1699	def getConfigBase(self):
1700	"""Returns the path that cfengine config files should live in"""
1701	if self.rDir is None:
1702	return None
1703	return "%s/inputs" % self.rDir
1704
1705	def _getFsPtr(self):
1706	# Strip scheme from URL
1707	path = self.svnroot[self.svnroot.find("://")+3:]
1708	rep = repos.svn_repos_open(path, self.pool)
1709	fs_ptr = repos.svn_repos_fs(rep)
1710	return fs_ptr
1711
1712	def _checkForModified(self, cDir, recursed=False):
1713	"""Performs svn actions on changed files in the specified directory"""
1714	# Nothing to check if no repository is checked out
1715	if self.rDir is None:
1716	return
1717
1718	# Don't check the status of a read-only revision
1719	if self.mParentSession is None or self.mChangeset is None:
1720	log_warn("Cannot check status on a read-only revision!")
1721	return
1722
1723	if not recursed:
1724	self._needsCommit = False
1725
1726	# Recurse through if we were passed a list
1727	if type(cDir) == type([]):
1728	for d in cDir:
1729	self._checkForModified(d)
1730	return
1731
1732	self.lock.acquire()
1733	try:
1734	# Callback function to process status information
1735	def status_cb(epath, entry):
1736	# Check the text of the entry
1737	if entry.text_status == wc.svn_wc_status_unversioned:
1738	if epath.startswith(self.rDir):
1739	ename = epath[len(self.rDir)+1:]
1740	log_debug("Added %s in changeset %s" % \
1741	(ename, self.mChangeset))
1742	client.svn_client_add(epath, False, self.ctx, self.pool)
1743	self._needsCommit = True
1744	# Recurse if we added a directory
1745	if os.path.isdir(epath):
1746	self._checkForModified(epath)
1747	else:
1748	# Ensure the date property is set on files
1749	self.propset(epath, "svn:keywords", "Date")
1750	# Check if there are any other pending properties to
1751	# set
1752	path = epath
1753	if path in self.pendingProps.keys():
1754	for prop,value in self.pendingProps[path].items():
1755	self.propset(path, prop, value, False)
1756	elif entry.text_status == wc.svn_wc_status_modified or \
1757	entry.text_status == wc.svn_wc_status_added or \
1758	entry.text_status == wc.svn_wc_status_deleted or \
1759	entry.text_status == wc.svn_wc_status_replaced or \
1760	entry.text_status == wc.svn_wc_status_merged:
1761	self._needsCommit = True
1762	elif entry.text_status == wc.svn_wc_status_ignored:
1763	# Ignore it!
1764	pass
1765	elif entry.text_status != wc.svn_wc_status_normal:
1766	log_debug("%s (%s) has bad state in changeset %s!" % \
1767	(epath, entry.text_status, self.mChangeset))
1768	# Now look at the properties of the entry
1769	if entry.prop_status == wc.svn_wc_status_modified or \
1770	entry.prop_status == wc.svn_wc_status_added or \
1771	entry.prop_status == wc.svn_wc_status_deleted or \
1772	entry.prop_status == wc.svn_wc_status_replaced or \
1773	entry.prop_status == wc.svn_wc_status_merged:
1774	self._needsCommit = True
1775	# Ensure that ccs-revision files are always ignored
1776	if os.path.isdir(epath):
1777	if not self.hasIgnore(epath, "ccs-revision"):
1778	self.propadd(epath, "svn:ignore", "ccs-revision")
1779	self._needsCommit = True
1780
1781	# Check the status
1782	client.svn_client_status(cDir, self.mCurRev, status_cb, \
1783	True, True, False, False, self.ctx, self.pool)
1784	finally:
1785	self.lock.release()
1786
1787	def propadd(self, path, prop, value, canDefer=True, sep="\n"):
1788	"""Adds the specified value to the property on the specified path"""
1789
1790	# Use the svn repo directly if nothing is checked out
1791	if self.rDir is None:
1792	base = self.svnroot
1793	else:
1794	base = self.rDir
1795
1796	try:
1797	if not path.startswith(base):
1798	filename = "%s/%s" % (base, path)
1799	else:
1800	filename = path
1801	filename = filename.rstrip("/")
1802	self.lock.acquire()
1803	try:
1804	rev = core.svn_opt_revision_t()
1805	rev.kind = core.svn_opt_revision_head
1806	eprop = client.svn_client_propget(prop, filename, rev, False, \
1807	self.ctx, self.pool)
1808	if eprop == {}:
1809	# No existing property set
1810	self.propset(path, prop, value, canDefer)
1811	else:
1812
1813	existing = str(eprop.values()[0]).strip()
1814	self.propset(path, prop, \
1815	"%s%s%s" % (existing, sep, value), canDefer)
1816	finally:
1817	self.lock.release()
1818	except:
1819	log_error("Could not add '%s' to %s on %s" % \
1820	(value, prop, filename), sys.exc_info())
1821
1822	def propset(self, path, prop, value, canDefer=True):
1823	"""Sets the specified property on the specified path"""
1824
1825	if self.mParentSession is None or self.mChangeset is None:
1826	raise ccs_revision_error("Cannot set property on read-only " \
1827	"revision")
1828
1829	# Use the svn repo directly if nothing is checked out
1830	if self.rDir is None:
1831	base = self.svnroot
1832	else:
1833	base = self.rDir
1834
1835	try:
1836	if not path.startswith(base):
1837	filename = "%s/%s" % (base, path)
1838	else:
1839	filename = path
1840	filename = filename.rstrip("/")
1841	self.lock.acquire()
1842	try:
1843	client.svn_client_propset(prop, value, filename, False, \
1844	self.pool)
1845	finally:
1846	self.lock.release()
1847	except core.SubversionException:
1848	# Add to a list of pending properties that we'll try and set on
1849	# commit after adding the file. This covers the case where a
1850	# caller tries to set a property on an as yet unversioned file
1851	if filename in self.pendingProps.keys():
1852	self.pendingProps[filename][prop] = value
1853	else:
1854	self.pendingProps[filename] = {prop:value}
1855	log_debug("Deferred propset on %s" % filename, sys.exc_info())
1856	except:
1857	log_error("Could not set %s to '%s' on %s" % \
1858	(prop, value, filename), sys.exc_info())
1859
1860	@registerEvent("revisionCreated")
1861	def checkin(self, message, paths=None):
1862	"""Checks in the changes to the repository with the specified message"""
1863	if self.rDir is None:
1864	raise ccs_revision_error("Cannot checkin. Not checked out!")
1865
1866	if self.mParentSession is None or self.mChangeset is None:
1867	raise ccs_revision_error("Cannot checkin a read-only revision")
1868
1869	# Default to the whole repository
1870	if paths is None:
1871	paths = [self.rDir]
1872	if type(paths) != type([]):
1873	paths = [paths]
1874
1875	# Check status of working directory and add / del files etc
1876	self._checkForModified(paths)
1877	if not self._needsCommit:
1878	# Nothing changed
1879	return -1
1880
1881	self.lock.acquire()
1882	try:
1883	i = client.svn_client_commit(paths, False, self.ctx, self.pool)
1884	if i.revision < 0:
1885	return -1
1886	n = self.saveRevProps(i.revision, message)
1887	finally:
1888	self.lock.release()
1889	if n<0:
1890	# Nothing changed
1891	return -1
1892
1893	triggerEvent(self.mParentSession.session_id, "revisionCreated", \
1894	revision_no=n)
1895	log_info("Committed revision %s to version control" % i.revision)
1896
1897	return n
1898
1899	def saveRevProps(self, r, message=""):
1900	"""Saves customised properties against each revision
1901
1902	These properties are used to help keep track of how the database/svn
1903	repository changes match up.
1904	"""
1905
1906	if r is None:
1907	# Nothing changed
1908	return -1
1909
1910	rev = core.svn_opt_revision_t()
1911	rev.kind = core.svn_opt_revision_number
1912	rev.value.number = int(r)
1913
1914	# Set the log message if specified
1915	if message != "":
1916	try:
1917	self.lock.acquire()
1918	try:
1919	client.svn_client_revprop_set("svn:log", message, \
1920	self.svnroot, rev, False, self.ctx, self.pool)
1921	finally:
1922	self.lock.release()
1923	except:
1924	(type, value, tb) = sys.exc_info()
1925	log_warn("Could not set log property on revision %s - %s" % \
1926	(r, value), (type, value, tb))
1927
1928	# Set the author property on the checkin
1929	try:
1930	self.lock.acquire()
1931	try:
1932	client.svn_client_revprop_set("svn:author", \
1933	str(self.mParentSession.username), self.svnroot, rev, \
1934	False, self.ctx, self.pool)
1935	finally:
1936	self.lock.release()
1937	except:
1938	(type, value, tb) = sys.exc_info()
1939	log_warn("Could not set author property on revision %s - %s" % \
1940	(r, value), (type, value, tb))
1941
1942	# Record the changeset that triggered this revision
1943	try:
1944	self.lock.acquire()
1945	try:
1946	client.svn_client_revprop_set("ccs:changeset", "%s" % \
1947	self.mChangeset, self.svnroot, rev, \
1948	False, self.ctx, self.pool)
1949	finally:
1950	self.lock.release()
1951	except:
1952	(type, value, tb) = sys.exc_info()
1953	log_warn("Could not set changeset property on " \
1954	"revision %s - %s" % (r, value), (type, value, tb))
1955
1956	return r
1957
1958	def fileExists(self, path):
1959	"""Checks if the specified file exists in the repository"""
1960
1961	# Use the svn repo directly if nothing is checked out
1962	if self.rDir is None:
1963	base = self.svnroot
1964	else:
1965	base = self.rDir
1966
1967	try:
1968	if not path.startswith(base):
1969	filename = "%s/%s" % (base, path)
1970	else:
1971	filename = path
1972	filename = filename.rstrip("/")
1973	self.lock.acquire()
1974	try:
1975	rev = core.svn_opt_revision_t()
1976	rev.kind = core.svn_opt_revision_head
1977	e = client.svn_client_ls(filename, rev, False, \
1978	self.ctx, self.pool)
1979	finally:
1980	self.lock.release()
1981	if len(e) > 0:
1982	return True
1983	except core.SubversionException:
1984	(type, value, tb) = sys.exc_info()
1985	if value[0].find("non-existent") == -1:
1986	log_warn("Failed to check existance of file: %s" % filename, \
1987	(type, value, tb))
1988	return False
1989	except:
1990	log_warn("Failed to check existance of file: %s" % filename, \
1991	sys.exc_info())
1992	return False
1993
1994	return False
1995
1996	def revinfo(self, revno):
1997	if revno in self._revinfocache.keys():
1998	return self._revinfocache[revno]
1999
2000	self.lock.acquire()
2001	try:
2002	rev = core.svn_opt_revision_t()
2003	rev.kind = core.svn_opt_revision_number
2004	rev.value.number = int(revno)
2005	props = client.svn_client_revprop_list(self.svnroot, rev, \
2006	self.ctx, self.pool)
2007	finally:
2008	self.lock.release()
2009	if len(props) != 2:
2010	return {}
2011	info = {}
2012	info["number"] = props[1]
2013	for prop,value in props[0].items():
2014	info[prop.strip("svn:")] = str(value)
2015	self._revinfocache[revno] = info
2016	return info
2017
2018	def ls(self, path, revno=None):
2019	"""Returns a list of entries in the specified directory"""
2020
2021	# Use the svn repo directly if nothing is checked out
2022	if self.rDir is None:
2023	base = self.svnroot
2024	else:
2025	base = self.rDir
2026
2027	try:
2028	if not path.startswith(base):
2029	filename = "%s/%s" % (base, path)
2030	else:
2031	filename = path
2032	filename = filename.rstrip("/")
2033	self.lock.acquire()
2034	try:
2035	if revno is None or revno=="":
2036	rev = core.svn_opt_revision_t()
2037	rev.kind = core.svn_opt_revision_head
2038	else:
2039	rev = core.svn_opt_revision_t()
2040	rev.kind = core.svn_opt_revision_number
2041	rev.value.number = int(revno)
2042	e = client.svn_client_ls(filename, rev, False, self.ctx, \
2043	self.pool)
2044	finally:
2045	self.lock.release()
2046	if len(e) > 0:
2047	if revno is None:
2048	revno = self.getYoungestRevision()
2049	entries = []
2050	for entry,details in e.items():
2051	t = {}
2052	if path.endswith(entry):
2053	t["name"] = "%s/%s" % (os.path.dirname(path), entry)
2054	else:
2055	t["name"] = "%s/%s" % (path, entry)
2056	t["created_rev"] = self.revinfo(details.created_rev)
2057	t["kind"] = details.kind
2058	t["last_author"] = details.last_author
2059	t["size"] = details.size
2060	t["time"] = details.time/1000000
2061	entries.append(t)
2062	return (revno, entries)
2063	except:
2064	log_warn("Could not list directory: %s" % filename, sys.exc_info())
2065
2066	return (-1, [])
2067
2068	def getYoungestRevision(self, path=""):
2069	"""Returns the number of the youngest revision in the repository"""
2070	revno = -1
2071
2072	try:
2073	self.lock.acquire()
2074	try:
2075	# Strip scheme from URL
2076	fs_ptr = self._getFsPtr()
2077	revno = fs.youngest_rev(fs_ptr, self.pool)
2078	finally:
2079	self.lock.release()
2080	except:
2081	log_warn("Could not determine youngest revision", sys.exc_info())
2082
2083	return revno
2084
2085	def getLog(self, revno):
2086	"""Returns the log message for the specified revision"""
2087	log = ""
2088
2089	def rcvMessage(a,b,c,d,msg,f):
2090	log = msg.strip()
2091
2092	try:
2093	self.lock.acquire()
2094	try:
2095	rev = core.svn_opt_revision_t()
2096	rev.kind = core.svn_opt_revision_number
2097	rev.value.number = int(revno)
2098	client.svn_client_log([self.svnroot], rev, rev, False, \
2099	False, nmsg, self.ctx, self.pool)
2100	finally:
2101	self.lock.release()
2102	except:
2103	log_warn("Could not retrieve log for revision %s" % revno, \
2104	sys.exc_info())
2105
2106	return log
2107
2108	def getFile(self, path, revno=None):
2109	"""Returns the contents of the file at the specified path and rev"""
2110
2111	# Use the svn repo directly if nothing is checked out
2112	if self.rDir is None:
2113	base = self.svnroot
2114	else:
2115	base = self.rDir
2116
2117	try:
2118	if not path.startswith(base):
2119	filename = path[len(base)+1:]
2120	else:
2121	filename = path
2122	filename = filename.rstrip("/")
2123	self.lock.acquire()
2124	try:
2125	if revno is None or revno=="":
2126	revno = self.getYoungestRevision()
2127	else:
2128	revno = int(revno)
2129	fs_ptr = self._getFsPtr()
2130	root = fs.revision_root(fs_ptr, revno, self.pool)
2131	s = core.Stream(fs.file_contents(root, path, self.pool))
2132	contents = s.read()
2133	finally:
2134	self.lock.release()
2135	return contents
2136	except:
2137	log_warn("Could not retrieve file: %s" % filename, sys.exc_info())
2138
2139	return ""
2140
2141	def getProps(self, path, revno=None):
2142	"""Returns the properties set on the specified file"""
2143
2144	# Use the svn repo directly if nothing is checked out
2145	if self.rDir is None:
2146	base = self.svnroot
2147	else:
2148	base = self.rDir
2149
2150	try:
2151	if not path.startswith(base):
2152	filename = "%s/%s" % (base, path)
2153	else:
2154	filename = path
2155	filename = filename.rstrip("/")
2156	self.lock.acquire()
2157	try:
2158	if revno is None or revno=="":
2159	rev = core.svn_opt_revision_t()
2160	rev.kind = core.svn_opt_revision_head
2161	else:
2162	rev = core.svn_opt_revision_t()
2163	rev.kind = core.svn_opt_revision_number
2164	rev.value.number = int(revno)
2165	contents = client.svn_client_proplist(filename, rev, \
2166	False, self.ctx, self.pool)
2167	finally:
2168	self.lock.release()
2169	if len(contents) > 0:
2170	props = {}
2171	for prop,value in contents[0][1].items():
2172	props[prop] = str(value)
2173	return props
2174	except:
2175	log_warn("Could not retrieve properties: %s" % \
2176	filename, sys.exc_info())
2177
2178	return {}
2179
2180	def hasIgnore(self, path, value):
2181	"""Checks whether the specified ignore value is set on the path"""
2182
2183	props = client.svn_client_proplist(path, self.mCurRev, \
2184	False, self.ctx, self.pool)
2185	if len(props)<1:
2186	return False
2187
2188	(path, pdict) = props[0]
2189	if "svn:ignore" not in pdict.keys():
2190	return False
2191	if str(pdict["svn:ignore"]).find(value) == -1:
2192	return False
2193
2194	return True
2195
2196	def checkRepoStructure(self):
2197	"""Checks the repository has all the required base directories.
2198
2199	If a base directory is not present it is created. The base directories
2200	are:
2201	inputs/ Host/Service configuration files
2202
2203	Further hierarchy within each base directory is the responsibility of
2204	other modules.
2205	"""
2206
2207	if self.rDir is None:
2208	raise ccs_revision_error("Cannot check repository structure. " \
2209	"Not checked out!")
2210
2211	if self.mParentSession is None or self.mChangeset is None:
2212	log_warn("Cannot check repository structure on a read-only " \
2213	"revision")
2214	return
2215
2216	flag = 0
2217
2218	configsDir = self.getConfigBase()
2219	n = ensureDirExists(configsDir)
2220	self.lock.acquire()
2221	try:
2222	if n > 0:
2223	# Schedule Additions
2224	bDir = configsDir
2225	while n>1:
2226	bDir = os.path.dirname(configsDir)
2227	n-=1
2228	log_info("Created configuration directory (%s) within " \
2229	"repository" % configsDir)
2230	client.svn_client_add(bDir, False, self.ctx, self.pool)
2231	flag = 1
2232
2233	# Commit directory changes immediately
2234	if flag==1:
2235	self.checkin("checkRepoStructure created missing directories")
2236	#i = client.svn_client_commit([self.rDir], False, self.ctx, \
2237	# self.pool)
2238	#self.saveRevProps(i.revision, \
2239	# "checkRepoStructure created missing directories")
2240	finally:
2241	self.lock.release()
2242
2243	# Check for the script that updates the ccs-revision files
2244	filename = "%s/update-revisioninfo" % configsDir
2245	docreate=False
2246	doupdate=False
2247	if not os.path.exists(filename):
2248	docreate=True
2249	else:
2250	contents = file(filename, "r").read()
2251	if contents.find("svn info") == -1:
2252	doupdate = True
2253	if docreate or doupdate:
2254	self.lock.acquire()
2255	try:
2256	# Create the script
2257	fp = open(filename, "w")
2258	fp.write("""#!/bin/bash
2259	echo "Updating version information"
2260	for d in `find $1 -type d \| grep -v ".svn" \| xargs`; do
2261	version=`svn info $d/* \| grep "Last Changed Rev" \| awk '{print $4}' \| sort -nr \| head -n1`
2262	echo " $d -> $version"
2263	echo "$version" > $d/ccs-revision
2264	done
2265	# vim:set sw=4 ts=4 sts=4 et:
2266	""")
2267	fp.close()
2268	if docreate:
2269	client.svn_client_add(filename, False, self.ctx, self.pool)
2270	client.svn_client_propset(prop, value, filename, False, \
2271	self.pool)
2272	action = "Created"
2273	else:
2274	action = "Updated"
2275	i = client.svn_client_commit([self.rDir], False, self.ctx, \
2276	self.pool)
2277	self.saveRevProps(i.revision, \
2278	"%s update-revisioninfo script" % action)
2279	finally:
2280	self.lock.release()
2281
2282	def validateRepository(svnroot):
2283	"""Checks for the existance of a valid svn repository at svnroot"""
2284
2285	# Extract the path from the URL
2286	svnpath = svnroot[svnroot.find("://")+3:]
2287
2288	# Check directory exists
2289	if not os.path.exists(svnpath):
2290	log_debug("Specified repository path does not exist: %s" % svnpath)
2291	return False
2292
2293	# Check it looks a bit like a subversion repository
2294	fp = open("%s/README.txt" % svnpath, "r")
2295	if not fp:
2296	log_debug("No README.txt inside repository: %s" % svnpath)
2297	return False
2298	tmp = fp.read().strip()
2299	fp.close()
2300	if not tmp.startswith("This is a Subversion repository"):
2301	log_debug("invalid README.txt in repository: %s" % svnpath)
2302	return False
2303
2304	# Final, master check, can we check it out, instantiate a revision class
2305	revision = ccs_revision(True, True)
2306	del revision
2307
2308	# All OK
2309	return True
2310
2311	def createRepository(svnroot):
2312	"""Initialises a blank repository and configures it for use by ccsd"""
2313
2314	# Look for the svnadmin utility
2315	fd = os.popen("/usr/bin/which svnadmin 2>/dev/null")
2316	path = fd.read().strip()
2317	if fd.close() != None:
2318	log_error("Cannot find svnadmin utility.")
2319	return False
2320
2321	# Check we can execute the svnadmin utilty
2322	if not os.access(path, os.X_OK):
2323	log_error("Cannot execute svnadmin utility.")
2324	return False
2325
2326	# Try and create the repository
2327	svnpath = svnroot[svnroot.find("://")+3:]
2328	fd = os.popen("%s create --fs-type fsfs %s 2>&1" % (path, svnpath))
2329	result = fd.read().strip()
2330	if fd.close() != None:
2331	log_error("svnadmin create failed. See tb log for more details.")
2332	log_tb(None, result)
2333	return False
2334
2335	# Add a pre-revprop-change hook to allow the daemon to set author
2336	# revision properties
2337	hook = "%s/hooks/pre-revprop-change" % svnpath
2338	fd = open(hook, "w")
2339	if not fd:
2340	log_warning("Unable to setup pre-revprop-change hook in new " \
2341	"repository!")
2342	else:
2343	fd.write("""#!/bin/bash
2344
2345	# PRE-REVPROP-CHANGE HOOK
2346
2347	# Created by the CRCnet Configuration System Daemon on
2348	# %s
2349
2350	# Allow all changes at this time
2351	exit 0
2352	""" % time.ctime())
2353	fd.close()
2354	os.chmod(hook, 0755)
2355
2356	# Success
2357	log_info("Subversion repository created at %s" % svnpath)
2358	return True
2359
2360	#####################################################################
2361	# Cfengine Integration Initialisation
2362	#####################################################################
2363	# Base CFengine paths
2364	_cfBaseDir = ""
2365	_cfInputDir = ""
2366	_svnroot = ""
2367	# Global information about template generation
2368	_hostTemplates = {}
2369	_networkTemplates = {}
2370	_templateDir = ""
2371	_templateModDir = ""
2372	_events = {}
2373	_templateStats = {}
2374	_statsLock = None
2375	# Global information about cfrun instances
2376	_cfrunStatus = {}
2377	_cfrunLock = None
2378	# Semaphore to control how many threads we initiate
2379	_threadLimit = None
2380
2381	def initCFengine():
2382	global _hostTemplates, _networkTemplates, _templateDir, _templateModDir
2383	global _events, _cfBaseDir, _cfInputDir, _svnroot, _statsLock, _threadLimit
2384	global _cfrunLock
2385
2386	# Retrieve template directory configuration
2387	_templateDir = config_get("cfengine", "template_dir")
2388	if _templateDir == None or _templateDir == "":
2389	log_fatal("Invalid template directory (%s)!" % _templateDir)
2390	_templateModDir = config_get("cfengine", "template_module_dir")
2391	if _templateModDir == None or _templateModDir == "":
2392	log_fatal("Invalid template module directory (%s)!" % _templateModDir)
2393
2394	# Retrieve cfengine configuration file directory configuration
2395	_cfBaseDir = config_get("cfengine", "cfbase_dir")
2396	if _cfBaseDir == None or _cfBaseDir == "":
2397	log_fatal("Invalid cfengine base directory (%s)!" % _cfBaseDir)
2398	_cfInputDir = "%s/inputs" % _cfBaseDir
2399
2400	# Decide where the svn root is
2401	_svnroot = config_get("cfengine", "config_svnroot")
2402	if _svnroot == None or _svnroot == "":
2403	log_fatal("Configuration Data subversion repository not specified!")
2404	# Check that the svnroot is a valid svn URL
2405	if _svnroot.find("://")==-1:
2406	log_fatal("Invalid svnroot URL: %s" % _svnroot)
2407	# Let the revision class know where to get its data from
2408	ccs_revision.svnroot = _svnroot
2409	# Validate the repository and create it if it doesn't exist
2410	if not validateRepository(_svnroot):
2411	log_warn("No svn repository at specified path. Attempting " \
2412	"to create one.")
2413	if not createRepository(_svnroot):
2414	log_fatal("Could not create repository at %s" % _svnroot)
2415
2416	# Initialise the statistics lock
2417	_statsLock = threading.Lock()
2418
2419	# Initialise the cfrun lock
2420	_cfrunLock = threading.Lock()
2421
2422	# How many threads are we allowed to run at a time
2423	maxThreads = config_get("cfengine", "max_threads", DEFAULT_MAX_THREADS)
2424	suggestThreadpoolSize(maxThreads)
2425	_threadLimit = threading.Semaphore(maxThreads)
2426
2427	# Initialise the templates
2428	initTemplates()
2429
2430	# Register Cfengine Resource Server
2431	registerResource("cfengine", cfserver)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: