source: ccsd/trunk/crcnetd/_utils/ccsd_cfengine.py @ 1025

Last change on this file since 1025 was 1025, checked in by mglb1, 7 years ago

Fix typo in previous commit and import new symbols that are required
  • Property svn:keywords set to Id
File size: 81.4 KB
Line 
1# Copyright (C) 2006  The University of Waikato
2#
3# This file is part of crcnetd - CRCnet Configuration System Daemon
4#
5# CFengine Configuration Setup
6#
7# Manages the cfengine configuration for the configuration system. The two
8# primary tasks involved in this are:
9# - Configuration file generation from templates
10# - Controling cfengine runs and collecting output on demand
11#
12# Author:       Matt Brown <matt@crc.net.nz>
13# Version:      $Id$
14#
15# crcnetd is free software; you can redistribute it and/or modify it under the
16# terms of the GNU General Public License version 2 as published by the Free
17# Software Foundation.
18#
19# crcnetd is distributed in the hope that it will be useful, but WITHOUT ANY
20# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
21# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
22# details.
23#
24# You should have received a copy of the GNU General Public License along with
25# crcnetd; if not, write to the Free Software Foundation, Inc.,
26# 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
27import sys
28import os
29import os.path
30from stat import *
31import pysvn
32import imp
33import time
34import threading
35from twisted.web import resource, server
36from tempfile import mkdtemp
37from Cheetah.Template import Template
38
39from crcnetd._utils.ccsd_common import *
40from crcnetd._utils.ccsd_log import *
41from crcnetd._utils.ccsd_events import *
42from crcnetd._utils.ccsd_config import config_get, config_get_required
43from crcnetd._utils.ccsd_session import getSession, getSessionE
44from crcnetd._utils.ccsd_server import registerResource, exportViaXMLRPC, \
45        initThread, registerRecurring, suggestThreadpoolSize
46
47DEFAULT_MAX_THREADS = 10
48
49class ccs_template_error(ccsd_error):
50    pass
51class ccs_cfengine_error(ccsd_error):
52    pass
53
54#####################################################################
55# General Cfengine Integration Helper Functions
56#####################################################################
57def initTemplates():
58    global _hostTemplates, _networkTemplates, _templateDir, _templateModDir
59   
60    # Ensure output template module directories exist
61    ensureDirExists("%s/host" % _templateModDir)
62    ensureDirExists("%s/network" % _templateModDir)
63
64    # Ensure templates are compiled
65    rv = os.system("/usr/bin/cheetah compile -R --idir %s/host --odir " \
66            "%s/host --nobackup &>/dev/null" % (_templateDir, _templateModDir))
67    if rv != 0:
68        log_error("Unable to compile all host templates!")
69    rv = os.system("/usr/bin/cheetah compile -R --idir %s/network --odir " \
70            "%s/network --nobackup &>/dev/null" % \
71            (_templateDir, _templateModDir))
72    if rv != 0:
73        log_error("Unable to compile all network templates!")
74   
75    # Load template files
76    _hostTemplates = loadTemplates("%s/host" % _templateModDir)
77    log_info("Loaded %s host templates for cfengine." % len(_hostTemplates))
78    _networkTemplates = loadTemplates("%s/network" % _templateModDir)
79    log_info("Loaded %s network templates for cfengine." % \
80            len(_networkTemplates))
81
82    # Register for events
83    registerEvents(_hostTemplates)
84    registerEvents(_networkTemplates)
85
86def loadTemplates(moduleDir, baseStrip=""):
87    """Creates a dictionary of template modules from the specified dir"""
88   
89    # Dictionary of template objects
90    templates = {}
91   
92    if baseStrip == "":
93        baseStrip = moduleDir
94       
95    # Get a list of possible templates
96    if os.access(moduleDir, os.R_OK) != 1:
97        raise ccs_template_exception("Unable to access directory! - %s" % \
98                moduleDir)
99    ptempls = os.listdir(moduleDir)
100   
101    # Scan through the list and load valid modules
102    for tfile in ptempls:
103        # Ignore hidden files
104        if tfile.startswith("."):
105            continue
106        tfilename = "%s/%s" % (moduleDir, tfile)
107        # Recurse into directories
108        if os.path.isdir(tfilename):
109            templates.update(loadTemplates(tfilename, baseStrip))
110            continue
111        # Ignore non module files
112        if not tfile.endswith(".py"):
113            continue
114        # Load the template module
115        tname = os.path.basename(tfilename)[:-3]
116        if tname == "__init__":
117            # Ignore python framework stuff
118            continue
119        m = None
120        try:
121            m = imp.load_source(tname, tfilename)
122            # Don't want the module in the system module list
123            if tname in sys.modules.keys():
124                del sys.modules[tname]
125        except:
126            log_debug("Module import failed for %s template" % tname, \
127                    sys.exc_info())
128            pass
129        if not m:
130            log_error("Failed to import template: %s" % tname)
131            continue
132        # If an explicit output filename was specified use that, otherwise
133        # prepend path information and store the template filename.
134        path = "%s/" % os.path.dirname(tfilename[len(baseStrip)+1:])
135        if len(path) == 1:
136            path = ""
137        tclass = eval("m.%s" % tname)
138        m.fileName = "%s%s" % (path, getattr(tclass, "fileName", tname))
139        m.multiFile = getattr(tclass, "multiFile", False)
140        m.templateName = tname
141        # Store the template for future use, prepend path for uniqueness
142        templateID = "%s%s" % (path, tname)
143        if templateID in templates.keys():
144            log_error("Could not import duplicate template: %s" % tname)
145            continue
146        templates[templateID] = m
147   
148    # Return the templates
149    return templates
150
151def registerEvents(templates):
152    """Runs through the list of templates and registers callbacks"""
153   
154    for templateID, template in templates.items():
155        events = eval("template.%s.eventList" % template.templateName)
156        if events == [] or not events:
157            continue
158        for event in events:
159            registerCallback(templateID, event)
160   
161def registerCallback(templateID, eventName):
162    """Keeps our internal list of what templates want to be called"""
163
164    if eventName in _events.keys():
165        _events[eventName].append(templateID)
166        return
167
168    try:
169        catchEvent(eventName, processEvent)
170        _events[eventName] = [templateID]
171    except ccs_event_error:
172        log_warn("Template %s not bound to event %s!" % \
173                (templateID, eventName), sys.exc_info())
174
175def processEvent(eventName, host_id, session_id, **params):
176    """Callback process from the event module.
177   
178    This function is called when an event that a template has indicated
179    an interest in is fired. Look through our templates to find which
180    one is interested and call it.
181    """
182    global _hostTemplates, _networkTemplates
183    from crcnetd.modules.ccs_host import getHostName
184   
185    # Check session to see if a revision is active
186    session = getSessionE(session_id)
187    if session.revision == None:
188        return
189
190    host_name = getHostName(session_id, host_id)
191    hosts = {host_name:[],"network":[]}
192    # Fire all the templates attached to this event
193    for templateID in _events[eventName]:
194        # Process host templates if applicable
195        if host_id != -1 and templateID in _hostTemplates.keys():
196            hosts[host_name].append(templateID)
197        # Process any network templates
198        if templateID in _networkTemplates.keys():
199            hosts["network"].append(templateID)
200
201    # Fire off the generation job
202    generateTemplate(session_id, hosts)
203   
204@registerRecurring(60*10)
205def cleanTemplateStatus():
206    """Runs every ten minutes to clean up expired template status info"""
207    global _templateStats, _statsLock
208   
209    # If generation is not finished by this many minutes after initiation
210    # there is an error, stats are removed in hope of causing a Key error
211    # that will kill the errant threads
212    err_time = 60*60
213    # Statistics for finished generation runs are removed this many minutes
214    # after the run completes
215    expire_time = 60*30
216   
217    _statsLock.acquire()
218    try:
219        for key,stats in _templateStats.items():
220            if (time.time()-stats["initiated"]) > err_time:
221                # Still processing after one hour!?
222                log_error("Template generation (%s) was still active after " \
223                        "%d seconds!" % (key, err_time))
224                del _templateStats[key]
225                continue
226            if stats["finished"] == 0:
227                # Still in progress
228                continue
229            if (time.time()-stats["finished"]) > expire_time:
230                log_info("Removing template generation status (%s)" % key)
231                del _templateStats[key]
232    finally:
233        _statsLock.release()
234
235    return True
236   
237@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR)
238def getTemplateStatus(session_id, statsKey):
239    """Returns the status of a specified template generation run"""
240    global _templateStats, _statsLock
241
242    stats = None
243   
244    _statsLock.acquire()
245    try:
246        if statsKey not in _templateStats.keys():
247            raise ccs_cfengine_error("Invalid key: %s" % statsKey)
248        stats = dict(_templateStats[statsKey])
249    finally:
250        _statsLock.release()
251
252    return stats
253
254@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
255def generateTemplate(session_id, requested_templates={}):
256    """Generates the specified configuration templates
257
258    The requested_templates parameter should be a dictionary indexed by
259    host_name each entry should contain a list of the template ids that
260    should be generated for that host.
261   
262    If the template list for a host is empty all available templates will be
263    regenerated.
264    If the dictionary is completely empty then all hosts will have their
265    templates regenerated.
266
267    If an entry in the dictionary contains the key "network" it will be
268    interpreted as a list of network wide templates to regenerate. If this key
269    is absent then the network wide templates will be regenerated if the number
270    of host templates being regenerated is greater than one.
271   
272    All template processing is shunted off to another thread and the user is
273    returned a token that can be passed to future calls to getTemplateStatus
274    to retrieve the current status of template generation. Generation results
275    are stored for 30 minutes after the end of template generation.
276    """
277    global _hostTemplates, _networkTemplates, _templateStats, _statsLock
278    from crcnetd.modules.ccs_host import getHostList
279
280    # Setup the basic set of statistics about template generation
281    stats = {}
282    stats["planned"] = {}
283    stats["generated"] = {}
284    stats["initiated"] = time.time()
285    stats["setupprogress"] = 10
286    stats["generating"] = 0
287    stats["finished"] = 0
288    stats["error"] = None
289
290    # Work out what the user has requested us to generate
291    hosts = {}
292    host_names = requested_templates.keys()
293    if len(host_names) == 0:
294        host_names = [host["host_name"] for host in getHostList(session_id)]
295    for host in host_names:
296        if host == "network": continue
297        templates = []
298        if host in requested_templates.keys():
299            templates = requested_templates[host]
300        if len(templates) == 0:
301            templates = _hostTemplates.keys()
302        hosts[host] = templates
303    stats["planned"]["hosts"] = hosts
304    stats["generated"]["hosts"] = {}
305   
306    # Work out how many host templates we're going to be generating
307    total = 0
308    for tlist in hosts.values():
309        total += len(tlist)
310    stats["planned"]["total"] = total
311   
312    # Add network level templates if more than one host template is requested
313    # or they have been explicitly specified
314    if "network" in host_names:
315        stats["planned"]["network"] = requested_templates["network"]
316    elif total > 1:
317        stats["planned"]["network"] = _networkTemplates.keys()
318    else:
319        stats["planned"]["network"] = []
320    stats["planned"]["total"] += len(stats["planned"]["network"])
321    stats["generated"]["network"] = {}
322       
323    # Aquire the lock to deal with statistics
324    statsKey = token = createPassword(8)
325    _statsLock.acquire()
326    try:
327        _templateStats[statsKey] = stats
328    finally:
329        _statsLock.release()
330
331    # Fire off the new thread
332    initThread(processTemplates, session_id, statsKey)
333
334    # Return back to the user
335    return statsKey
336   
337def processTemplates(session_id, statsKey):
338    """Does the hardwork of generating templates in a thread"""
339    global _templateStats, _statsLock, _threadLimit
340    from crcnetd.modules.ccs_host import getHostList, ccs_host, \
341            getDistributions
342    from crcnetd._utils.ccsd_service import getServiceTemplateVars
343    from crcnetd.modules.ccs_asset import getAssetTypeTemplateVariables
344
345    log_info("Entered template processing thread (%s)" % statsKey)
346   
347    # Make sure we have a revision to insert files into
348    # Check session to see if a revision is active
349    session = getSessionE(session_id)
350    commit=0
351    if session.revision == None:
352        session.begin("Updated host configuration files", initiator="cfengine")
353        commit=1
354    outputDir = session.revision.getConfigBase()
355   
356    # More progress
357    _statsLock.acquire()
358    try:
359        _templateStats[statsKey]["setupProgress"] = 15
360    finally:
361        _statsLock.release()
362       
363    # Load all the data that we need to pass to the templates
364    variables = {}
365    hlist = {}
366    id2name = {}
367    for host in getHostList(session_id):
368        host = ccs_host(session_id, host["host_id"])
369        hostDetails = host.getTemplateVariables()
370        hlist[host["host_name"]] =  hostDetails
371        _statsLock.acquire()
372        try:
373            if _templateStats[statsKey]["setupProgress"] < 70:
374                _templateStats[statsKey]["setupProgress"] += 2
375        finally:
376            _statsLock.release()
377    variables["hosts"] = hlist
378    dlist = {}
379    for distrib in getDistributions(session_id):
380        dlist[distrib["distribution_id"]] = distrib
381    variables["distributions"] = dlist
382    variables["services"] = getServiceTemplateVars(session_id)
383    _statsLock.acquire()
384    try:
385        _templateStats[statsKey]["setupProgress"] = 80
386    finally:
387        _statsLock.release()
388    variables["date"] = time.ctime()
389    variables["domain"] = config_get_required("network", "domain")
390    variables["site_name"] = config_get_required("network", "site_name")
391    variables["server_name"] = config_get_required("network", "server_name")
392    variables["policy_ip"] = getIP(variables["server_name"])
393    variables["smtp_server"] = config_get_required("network", "smtp_server")
394    variables["admin_email"] = config_get_required("network", "admin_email")
395    _statsLock.acquire()
396    try:
397        _templateStats[statsKey]["setupProgress"] = 90
398    finally:
399        _statsLock.release()
400
401    variables["asset_types"] = getAssetTypeTemplateVariables(session_id)
402    variables["session_id"] = session_id
403
404    # Update the stats and get the list of hosts
405    _statsLock.acquire()
406    try:
407        planned = _templateStats[statsKey]["planned"]
408        _templateStats[statsKey]["generating"] = time.time()
409        _templateStats[statsKey]["setupProgress"] = 100
410    finally:
411        # Release the lock before proceeding
412        _statsLock.release()
413   
414    # Now generate each hosts template in a separate thread
415    for host,template_ids in planned["hosts"].items():
416        # See if we can start a new thread
417        _threadLimit.acquire()
418        try:
419            # Extract the host specific data for this template
420            hostData = variables["hosts"][host]
421            # Fire off the thread to process this host, the thread will release
422            # the semaphore as it is about to exit
423            # XXX: This relies on the thread behaving nicely, need to find a
424            # way to find out if it's been bad and exited without releasing
425            # the semaphore
426            initThread(processHostTemplates, statsKey, outputDir, hostData, \
427                    variables)
428        except:
429            log_error("Failed to start thread to process templates for " \
430                    "host: %s!" % host, sys.exc_info())
431            _threadLimit.release()
432           
433    # Wait until all the configuration files have been generated
434    hosts = list(planned["hosts"].keys())
435    hasError = False
436    hasTemplateError = False
437    while len(hosts) > 0:
438        # Wait a bit before trying again
439        time.sleep(1)
440        _statsLock.acquire()
441        try:
442            generated = _templateStats[statsKey]["generated"]["hosts"]
443            for host in hosts:
444                if not host in generated.keys():
445                    continue
446                # Check if it finished successfully
447                if generated[host]["finished"] > 0:
448                    # Remove from list
449                    hosts.remove(host)
450                    # Check if any of the templates failed to be output
451                    if generated[host]["error"] == "template":
452                        # Set the flag so we don't commit the revision
453                        hasTemplateError = True
454                else:
455                    # Check for fatal errors
456                    if generated[host]["error"] != "":
457                        # Set the flag and remove the host from processing
458                        hosts.remove(host)
459                        hasError = True
460                        continue
461        finally:
462            _statsLock.release()
463   
464    # Hosts are done, network templates now
465    for template_id in planned["network"]:
466        # See if we can start a new thread
467        _threadLimit.acquire()
468        try:
469            # Fire off the thread to process this template, the thread will
470            # release the semaphore as it is about to exit
471            # XXX: This relies on the thread behaving nicely, need to find a
472            # way to find out if it's been bad and exited without releasing
473            # the semaphore
474            initThread(processNetworkTemplate, statsKey, outputDir, \
475                    template_id, variables)
476        except:
477            log_error("Failed to start thread to process network template: " \
478                    "%s!" % template_id, sys.exc_info())
479            _threadLimit.release()
480
481    # Wait until all network templates are done
482    network = list(planned["network"])
483    while len(network) > 0:
484        # Wait a bit before trying again
485        time.sleep(1)
486        _statsLock.acquire()
487        try:
488            generated = _templateStats[statsKey]["generated"]["network"]
489            for template in network:
490                if not template in generated.keys():
491                    continue
492                # Check for errors
493                if generated[template]["error"] != "":
494                    hasTemplateError = True
495                # Remove from the list
496                network.remove(template)
497        finally:
498            _statsLock.release()       
499   
500    # Commit if neceesary and no template/host errors occured
501    rv = {"revision":None}
502    if commit and not (hasError or hasTemplateError):
503        rv = session.commit()
504       
505    # All done
506    _statsLock.acquire()
507    try:
508        _templateStats[statsKey]["finished"] = time.time()
509        _templateStats[statsKey]["revision"] = rv["revision"]
510        d = _templateStats[statsKey]["finished"] - \
511                _templateStats[statsKey]["initiated"]
512        if hasError:
513            _templateStats[statsKey]["error"] = "host"
514        elif hasTemplateError:
515            _templateStats[statsKey]["error"] = "template"
516    finally:
517        _statsLock.release()
518
519    log_info("Template generation completed in %0.3f seconds for %s" % \
520            (d, statsKey))
521    return True
522
523def processHostTemplates(statsKey, outputDir, hostData, networkData):
524    """Runs in a thread and generates the output files for the template"""
525    global _templateStats, _statsLock, _threadLimit, _hostTemplates
526    host_id = -1
527    host = "unknown"
528   
529    # It is imperative that we release the semaphore before returning from
530    # this function or we could potentially starve the calling thread of
531    # workers and we'd end up in a deadlock situation!
532    try:
533        # Work out the host_id and host_name of this host
534        host_id = hostData["host_id"]
535        host = hostData["host_name"]
536        hostPath = "%s/hosts/%s" % (outputDir, host)
537        start = time.time()
538        log_debug("Started host generation thread for %s" % host)
539        session = getSessionE(networkData["session_id"])
540       
541        # Retrieve the list of templates to generate and flag the start
542        _statsLock.acquire()
543        try:
544            planned = _templateStats[statsKey]["planned"]
545            generated = _templateStats[statsKey]["generated"]["hosts"]
546            generated[host] = {}
547            generated[host]["initiated"] = start
548            generated[host]["finished"] = 0
549            generated[host]["error"] = ""
550        finally:
551            _statsLock.release()
552
553        # Now that we have a set of templates to use do the actual generation
554        hadError = False
555        for tname in planned["hosts"][host]:
556            try:
557                tstart = time.time()
558                # Instantiate a template
559                template = _hostTemplates[tname]
560                t = eval("template.%s()" % template.templateName)
561                # Is it enabled on this host?
562                if not t.enabledOnHost(networkData["session_id"], host_id):
563                    continue
564                # Set up the variables we want to substitute in
565                t._searchList = [hostData, networkData]
566                # Generate the file
567                filename = "%s/%s" % (hostPath, template.fileName)
568                files = t.writeTemplate(filename, template.multiFile)
569                tend = time.time()
570                # Set properties
571                format = getattr(t, "highlightFormat", "")
572                if format != "":
573                    for file in files:
574                        session.revision.propset(file, "ccs:format", format)
575                # Record how long the file took to generate
576                _statsLock.acquire()
577                try:
578                    info = {"time":tend-tstart, "error":""}
579                    generated[host][tname] = info
580                finally:
581                    _statsLock.release()
582            except:
583                (type, value, tb) = sys.exc_info()
584                # Single template errors can be skipped over, nothing will
585                # get committed unless the user explicitly approves
586                _statsLock.acquire()
587                try:
588                   
589                    info = {"time":0, "error":value}
590                    generated[host][tname] = info
591                finally:
592                    _statsLock.release()
593                hadError = True
594                log_error("Failed to process template (%s) for host: %s" % \
595                        (tname, host), (type, value, tb))
596       
597        # Clean up
598        end = time.time()
599        log_debug("Completed host generation for %s in %0.3f seconds" % \
600                (host, (end-start)))
601        _statsLock.acquire()
602        try:
603            generated[host]["finished"] = end
604            # If a template failed make a note of it here
605            if hadError:
606                generated[host]["error"] = "template"
607        finally:
608            _statsLock.release()
609    except:
610        (type, value, tb) = sys.exc_info()
611        # Record the error for display to the user
612        _statsLock.acquire()
613        try:
614            generated = _templateStats[statsKey]["generated"]["hosts"]
615            if host not in generated.keys():
616                generated[host] = {}
617                generated[host]["initiated"] = 0
618            generated[host]["finished"] = time.time()
619            generated[host]["error"] = value
620        finally:
621            _statsLock.release()
622        # Log the error for the administrators record
623        log_error("Exception while processing host templates (%s)!" % \
624                host, (type, value, tb))
625   
626    # Release the semaphore
627    _threadLimit.release()
628    return
629
630def processNetworkTemplate(statsKey, outputDir, template_id, networkData):
631    """Runs in a thread and generates the output files for the template"""
632    global _templateStats, _statsLock, _threadLimit, _networkTemplates
633   
634    # It is imperative that we release the semaphore before returning from
635    # this function or we could potentially starve the calling thread of
636    # workers and we'd end up in a deadlock situation!
637    try:
638        log_debug("Started template generation thread for %s" % template_id)
639        tstart = time.time()
640        session = getSessionE(networkData["session_id"])
641        # Instantiate a template
642        template = _networkTemplates[template_id]
643        t = eval("template.%s()" % template.templateName)
644        # Set up the variables we want to substitute in
645        t._searchList = [networkData]
646        # Generate the file
647        filename = "%s/%s" % (outputDir, template.fileName)
648        files = t.writeTemplate(filename, template.multiFile)
649        tend = time.time()
650        # Set properties
651        format = getattr(template, "highlightFormat", "")
652        if format != "":
653            for file in files:
654                session.revision.propset(file, "ccs:format", format)
655        # Record how long the file took to generate
656        _statsLock.acquire()
657        try:
658            generated = _templateStats[statsKey]["generated"]["network"]
659            info = {"time":tend-tstart, "error":""}
660            generated[template_id] = info
661        finally:
662            _statsLock.release()
663        log_debug("Completed template generation for %s in %0.3f seconds" \
664                % (template_id, (tend-tstart)))
665    except:
666        (type, value, tb) = sys.exc_info()
667        # Skip over the error, nothing will get committed unless the user
668        # explicitly approves
669        _statsLock.acquire()
670        try:
671            generated = _templateStats[statsKey]["generated"]["network"]
672            info = {"time":0, "error":value}
673            generated[template_id] = info
674        finally:
675            _statsLock.release()
676        log_error("Failed to process network template (%s)" % template_id, \
677                (type, value, tb))
678   
679    # Release the semaphore
680    _threadLimit.release()
681    return
682
683#####################################################################
684# Template Mixin
685#####################################################################
686class ccs_template(Template):
687    """Config System Template Processor
688   
689    All templates that expect to be processed by the configuration system
690    must derive from this class. It provides helper functions that are required
691    to determine where and when each template should be generated and how to
692    store it to disk.
693    """
694
695    def __init__(self):
696        """Initialise the class
697
698        You must call this method from your subclass
699        """
700        # Call Cheetah's init
701        Template.__init__(self)
702
703    def writeTemplate(self, filename, multiFile):
704        """Writes the template output to the specified file"""
705       
706        files = []
707       
708        # Ensure the output directory exists
709        ensureDirExists(os.path.dirname(filename))
710
711        # Get the template contents
712        template = self.writeBody().strip()
713        if not template.endswith("\n"): template += "\n"
714
715        # Write it out to a file
716        if not multiFile:
717            f = open(filename, "w")
718            f.write(template)
719            f.close()
720            files.append(filename)
721        else:
722            # Handle templates that generate multiple output files
723            lines = template.split("\n")
724            f = None
725            for line in lines:
726                if line.startswith(".newfile"):
727                    # New file starting, close previous file
728                    if f is not None: f.close()
729                    # Open new file
730                    parts = line.split(" ")
731                    if len(parts) != 2:
732                        raise ccs_template_error("Invalid multifile template!")
733                    fname = "%s%s" % (filename, parts[1])
734                    f = open(fname, "w")
735                    files.append(fname)
736                    continue
737                elif f is not None:
738                    # Write the line out
739                    f.write("%s\n" % line)
740            # Close the file
741            if f is not None: f.close() 
742
743        return files
744
745    def __str__(self):
746        return self.writeBody()
747   
748    def enabledOnHost(self, session_id, host_id):
749        """Returns true if the template is applicable to the specified host
750
751        By default this method looks to see if the template has defined a
752        serviceName parameter. If it has then the function checks to see if
753        that service is enabled on the specified host. If it is not the
754        function returns False.
755       
756        This function may be overriden by other classes/templates if you want
757        to implement more logic than the default implementation provides.
758        """
759        serviceName = getattr(self, "serviceName", None)
760        if serviceName is None:
761            return True
762
763        from crcnetd.modules.ccs_host import ccs_host
764        host = ccs_host(session_id, host_id)
765        return host.hasServiceEnabledByName(serviceName)
766
767    def getTemplateVariables(self):
768        """Returns a dictionary of variables that can be used by the template
769
770        The dictionary is passed to Cheetah's searchList so that it's entries
771        can be used as placeholders in the template.
772
773        This function retains an empty list. You should override this in your
774        implementing class.
775        """
776        return []
777
778#####################################################################
779# Functions to maintain / deal with hosts
780#####################################################################
781def getCfengineHostStatus(session_id, host):
782    """Returns a dictionary describing the state of the host configuration"""
783
784    status = {}
785   
786    # Check keys exist
787    revision = ccs_revision(checkout=False)
788   
789    sshkeydir = "inputs/sshkeys/%s" % host["host_name"]
790    cfkeydir = "ppkeys"
791   
792    # SSH keys
793    status["ssh_key_status"] = STATUS_OK
794    status["ssh_key_text"] = ""
795    for key in ["dsa", "rsa"]:
796        if not revision.fileExists("%s/ssh_host_%s_key" % (sshkeydir, key)):
797            status["ssh_key_status"] = STATUS_CRITICAL
798            status["ssh_key_text"] += "%s private key missing, " % \
799                    key.upper()
800        if not revision.fileExists("%s/ssh_host_%s_key.pub" % \
801                (sshkeydir, key)):
802            status["ssh_key_status"] = STATUS_CRITICAL
803            status["ssh_key_text"] += "%s private key missing, " % \
804                    key.upper()
805    if status["ssh_key_text"]!="":
806        status["ssh_key_text"]  = status["ssh_key_text"][:-2]
807   
808    # Cfengine keys
809    status["cfengine_key_status"] = STATUS_OK
810    status["cfengine_key_text"] = ""
811    if not revision.fileExists("%s/root-%s.priv" % \
812            (cfkeydir, host["ip_address"])):
813        status["cfengine_key_status"] = STATUS_CRITICAL
814        status["cfengine_key_text"] += "CFengine private key missing"
815    if not revision.fileExists("%s/root-%s.pub" % \
816            (cfkeydir, host["ip_address"])):
817        status["cfengine_key_status"] = STATUS_CRITICAL
818        if status["cfengine_key_text"]!="": status["cfengine_key_text"]+=", "
819        status["cfengine_key_text"] += "CFengine public key missing"
820
821    return status
822   
823@catchEvent("hostAdded")
824def hostAddedCB(eventName, host_id, session_id, **params):
825    """Callback function to setup host configuration upon host creation"""
826
827    session = getSession(session_id)
828    if session is None:
829        return
830
831    # Create keys for the host
832    createSSHHostKeys(session_id, host_id)
833    createCfengineHostKeys(session_id, host_id)
834
835@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
836def createSSHHostKeys(session_id, host_id):
837    """Creates a set of SSH keys for the specified host"""
838    from crcnetd.modules.ccs_host import ccs_host
839    session = getSessionE(session_id)
840   
841    host = ccs_host(session_id, host_id)
842   
843    # Check for existing changeset
844    commit = 0
845    if session.changeset == 0:
846        session.begin("Created SSH keys for %s" % host["host_name"])
847        commit = 1
848   
849    try:
850        # Determine directories to store things in
851        revdir = session.revision.getConfigBase()
852        sshkeydir = "%s/sshkeys/%s" % (revdir, host["host_name"])
853        ensureDirExists(sshkeydir)
854       
855        # Generate the keys
856        genSSHKey(sshkeydir, "rsa")
857        genSSHKey(sshkeydir, "dsa")   
858    except:
859        (type, value, tb) = sys.exc_info()
860        log_error("Failed to generate SSH keys for %s! - %s" % \
861                (host["host_name"], value))
862        # Rollback any changes
863        if commit == 1:
864            session.rollback()
865        return False
866
867    # Commit changeset if necessary
868    if commit==1:
869        session.commit()
870       
871    return True
872
873@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
874def createCfengineHostKeys(session_id, host_id):
875    """Creates a set of Cfengine keys for the specified host"""
876    from crcnetd.modules.ccs_host import ccs_host
877    session = getSessionE(session_id)
878   
879    host = ccs_host(session_id, host_id)
880   
881    # Check for existing changeset
882    commit = 0
883    if session.changeset == 0:
884        session.begin("Created CFengine keys for %s" % host["host_name"])
885        commit = 1
886   
887    try:
888        # Determine directories to store things in
889        workdir = session.revision.getWorkingDir()
890        cfkeydir = "%s/ppkeys" % (workdir)
891        ensureDirExists(cfkeydir)
892       
893        # Generate the keys
894        genCfKey(cfkeydir, host["ip_address"])
895    except:
896        (type, value, tb) = sys.exc_info()
897        log_error("Failed to generate CFengine keys for %s! - %s" % \
898                (host["host_name"], value))
899        # Rollback any changes
900        if commit == 1:
901            session.rollback()
902        return False
903
904    # Commit changeset if necessary
905    if commit==1:
906        session.commit()
907       
908    return True
909
910def genSSHKey(dir, type):
911    """Generates a SSH host key of the specified type
912
913    dir specifies the directory to place the key in
914    """
915    filename = "ssh_host_%s_key" % type
916    key = "%s/%s" % (dir, filename)
917    pubkey = "%s.pub" % key
918   
919    # Try and avoid needless recreating keys
920    if os.path.exists(key) and os.path.exists(pubkey):
921        log_debug("Skipping SSH key creation in %s. " \
922                "Keys already exist!" % dir)
923        return
924    try:
925        os.remove(key)
926        os.remove(pubkey)
927    except:
928        pass
929       
930    fh = os.popen("cd %s &>/dev/null && /usr/bin/ssh-keygen -q -f %s " \
931            "-N '' -t %s || echo \"unable to access %s\"" % \
932            (dir, filename, type, dir))
933    output = fh.readlines()
934    rv = fh.close()
935    if rv != None or (len(output)>0 and output[0].startswith("unable")):
936        raise ccs_cfengine_error("Could not create SSH %s host key: %s" % \
937                (type, "".join(output)))
938   
939def genCfKey(dir, host_ip):
940    """Generates a Cfengine host key pair
941
942    dir specifies the directory to place the key in
943    """
944    filename = "root-%s" % host_ip
945    key = "%s/%s.priv" % (dir, filename)
946    pubkey = "%s/%s.pub" % (dir, filename)
947
948    # Try and avoid needless recreating keys
949    if os.path.exists(key) and os.path.exists(pubkey):
950        log_debug("Skipping Cfengine key creation in %s. " \
951                "Keys already exist!" % dir)
952        return
953    try:
954        os.remove(key)
955        os.remove(pubkey)
956    except:
957        pass
958
959    fh = os.popen("cd %s &>/dev/null && /usr/sbin/cfkey -f %s || echo " \
960            "\"unable to access %s\"" % (dir, filename, dir))
961    output = fh.readlines()
962    rv = fh.close()
963    if rv != None or (len(output)>0 and output[0].startswith("unable")):
964        raise ccs_cfengine_error("Could not create CFengine host key: %s" % \
965                "".join(output))
966   
967def getActiveRevision(hostname=None):
968    """Returns the active revision for the cfengine input directory
969
970    If a hostname is specified it looks in the hosts/<hostname> directory
971    specifically
972    """
973    global _cfInputDir
974
975    if hostname is None:
976        dir = _cfInputDir
977    else:
978        dir = "%s/hosts/%s" % (_cfInputDir, hostname)
979
980    # Get the revision number of the checked out revision
981    try:
982        contents = open("%s/ccs-revision" % dir, "r").read()
983        return int(contents.strip())
984    except: 
985        log_debug("Could not read ccs-revision", sys.exc_info())
986        pass
987   
988    # No checked out revision
989    return -1
990
991def getGeneratedRevision(hostname=None):
992    """Returns the latest revision that has been generated
993
994    If a hostname is specified it looks in the hosts/<hostname> directory
995    specifically
996    """
997   
998    if hostname is None:
999        dir = ""
1000    else:
1001        dir = "inputs/hosts/%s" % hostname
1002   
1003    revision = ccs_revision(checkout=False)
1004    return revision.getYoungestRevision(dir)
1005
1006@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR)
1007def getCfRunLogs(session_id):
1008    """Returns the output of the last cfrun execution for all active hosts"""
1009    from crcnetd.modules.ccs_host import getHostList, getHostID
1010    from crcnetd.modules.ccs_status import getHostStatus, ccs_status_error, \
1011            ccs_host_status
1012    global _cfBaseDir, _cfrunStatus
1013   
1014    hosts = {}
1015   
1016    outputDir = "%s/outputs" % _cfBaseDir
1017    domain = config_get_required("network", "domain")
1018
1019    # Get a list of active hosts
1020    for host in getHostList(session_id):
1021        # Skip hosts that are disabled
1022        if not host["host_active"]:
1023            continue
1024        # Store enabled hosts
1025        hostname = host["host_name"]
1026        hosts[hostname] = host
1027        # Retrieve the information about it
1028        filename = "%s/%s.%s" % (outputDir, host["host_name"], domain)
1029        hosts[hostname]["last_run"] = -1
1030        hosts[hostname]["age"] = ""
1031        hosts[hostname]["age_rounded"] = ""
1032        hosts[hostname]["contents"] = []
1033        hosts[hostname]["no_lines"] = 0
1034        try:
1035            hostStatus = getHostStatus(hostname)
1036        except ccs_status_error:
1037            # Create a blank status entry
1038            host_id = getHostID(ADMIN_SESSION_ID, hostname)
1039            hostStatus = ccs_host_status(ADMIN_SESSION_ID, host_id)
1040        hosts[hostname]["active_rev"] = hostStatus.activeRevision
1041        hosts[hostname]["generated_rev"] = hostStatus.generatedRevision
1042        hosts[hostname]["operating_rev_status"] = \
1043                hostStatus.operatingRevisionStatus
1044        hosts[hostname]["operating_rev_text"] = \
1045                hostStatus.operatingRevisionText
1046        hosts[hostname]["operating_rev_hint"] = \
1047                hostStatus.operatingRevisionHint
1048        if hostStatus.infoUpdatedAt > 0:
1049            hosts[hostname]["operating_rev"] = int(hostStatus.operatingRevision)
1050            try:
1051                o = int(hosts[hostname]["operating_rev"])
1052                a = int(hosts[hostname]["active_rev"])
1053                g = int(hosts[hostname]["generated_rev"])
1054                if o > g:
1055                    hosts[hostname]["operating_rev_status"] = "critical"
1056                    hosts[hostname]["operating_rev_text"] = "Invalid Revision"
1057                    hosts[hostname]["operating_rev_hint"] = "The operating " \
1058                            "revision is greater than the newest generated " \
1059                            "revision!"
1060                elif o != a:
1061                    hosts[hostname]["operating_rev_status"] = "warning"
1062                    hosts[hostname]["operating_rev_text"] = ""
1063                    hosts[hostname]["operating_rev_hint"] = "The operating " \
1064                            "revision does not match the desired active " \
1065                            "revision!"                   
1066            except: pass
1067        else:
1068            hosts[hostname]["operating_rev"] = ""
1069        if hostname in _cfrunStatus.keys():
1070            hosts[hostname]["run_status"] = _cfrunStatus[hostname]["status"]
1071            hosts[hostname]["run_age"] = time.strftime("%d/%m/%Y %H:%M:%S", \
1072                    time.localtime(_cfrunStatus[hostname]["time"]))
1073            hosts[hostname]["run_age_rounded"] = roundTime(time.time() - \
1074                    _cfrunStatus[hostname]["time"])
1075        else:
1076            hosts[hostname]["run_status"] = ""
1077            hosts[hostname]["run_age"] = ""
1078            hosts[hostname]["run_age_rounded"] = ""
1079        hosts[hostname]["error"] = ""
1080        if not os.path.exists(filename):
1081            # No cfrun output for this host
1082            continue
1083        try:
1084            stat = os.stat("%s/%s.%s" % (outputDir, host["host_name"], domain))
1085            hosts[hostname]["age"] = time.strftime("%d/%m/%Y %H:%M:%S", \
1086                    time.localtime(stat[ST_MTIME]))
1087            hosts[hostname]["age_rounded"] = roundTime(time.time() - \
1088                    stat[ST_MTIME])
1089            hosts[hostname]["last_run"] = stat[ST_MTIME]
1090            hosts[hostname]["contents"] = open(filename).read()
1091            hosts[hostname]["no_lines"] = \
1092                    len(hosts[hostname]["contents"].split("\n"))
1093        except:
1094            (type, value, tb) = sys.exc_info()
1095            log_error("Failed to retrieve CfRun log for %s" % hostname, \
1096                    (type, value, tb))
1097            hosts[hostname]["error"] = value
1098            continue
1099
1100    # Return it
1101    return hosts
1102
1103@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
1104def pushConfig(session_id, hosts, params=[]):
1105    """Initiates a cfrun instance to push a new config out to a host"""
1106    global _threadLimit, _cfrunStatus, _cfrunLock
1107
1108    # PHP sends in an associative array we want the values only
1109    hosts = hosts.values()
1110
1111    # Loop through the specified hosts and mark them as pending
1112    _cfrunLock.acquire()
1113    try:
1114        try:
1115            for host in hosts:
1116                _cfrunStatus[host] = {"status":"pending","time":time.time()}
1117        except:
1118            log_error("Exception setting up config push!", sys.exc_info())
1119            raise ccs_cfengine_error("Could not initiate config push!")
1120    finally:
1121        _cfrunLock.release()
1122   
1123    # Start the threads
1124    initThread(startCfRunThreads, hosts, params)
1125    return True
1126
1127def startCfRunThreads(hosts, params):
1128
1129    # Loop through the hosts and start threads to do the cfrun stuff
1130    for host in hosts:
1131        _threadLimit.acquire()
1132        try:
1133            initThread(doCfrun, host, params)
1134        except:
1135            log_error("Failed to start cfrun instance for host: %s!" % \
1136                    host, sys.exc_info())
1137            _threadLimit.release()
1138    return True
1139
1140def doCfrun(host_name, params=[]):
1141    """Executes a cfrun instance to update the specified host
1142
1143    This function expects to be called in a thread and will take care of
1144    releasing the semaphore when it finishes.
1145    """
1146    global _threadLimit, _cfrunStatus, _cfrunLock, _cfBaseDir
1147   
1148    outputDir = "%s/outputs" % _cfBaseDir
1149    domain = config_get_required("network", "domain")
1150    cmdline = ""
1151    filename = ""
1152   
1153    # Make sure that any errors are captured so that we can't exit without
1154    # releasing the semaphore
1155    try:
1156        # Record that this host is being processed
1157        _cfrunLock.acquire()
1158        try:
1159            if host_name in _cfrunStatus.keys():
1160                if _cfrunStatus[host_name]["status"] != "pending":
1161                    # This host is already being processed!
1162                    _threadLimit.release()
1163                    return
1164            _cfrunStatus[host_name] = {"status":"running","time":time.time()}
1165        finally:
1166            _cfrunLock.release()
1167        # Ouput filename
1168        filename = "%s/%s.%s" % (outputDir, host_name, domain)
1169        try:
1170            os.unlink(filename)
1171        except:
1172            log_warn("Could not remove old logfile before cfrun! - %s" % \
1173                    filename, sys.exc_info())
1174        # Build the command line
1175        cmdline = "/usr/sbin/cfrun %s" % host_name
1176        if len(params) > 0:
1177            cmdline += " -- "
1178            for i,param in params.items(): cmdline += "%s " % param
1179        # Execute the command
1180        fp = os.popen(cmdline, "r")
1181        output = fp.read().strip()
1182        rv = fp.close()
1183        if rv is not None:
1184            code="failed"
1185        else:
1186            code="succeeded"
1187        # Retrieve the actual cfrun output
1188        try:
1189            contents = open(filename, "r").read()
1190        except:
1191            log_warn("Could not read logfile from cfrun! - %s" % filename, \
1192                    sys.exc_info())
1193            contents = ""
1194        # Prepend the output from the execution to the log
1195        fp = open(filename, "w")
1196        fp.write("%s\n" % ("*"*80))
1197        fp.write("Command execution %s!\n" % code)
1198        fp.write("Cmdline: %s\n" % cmdline)
1199        if len(output) > 0:
1200                fp.write("\n%s\n" % output)
1201        fp.write("%s\n\n" % ("*"*80))
1202        fp.write(contents)
1203        fp.close()
1204        # Record that this host is no longer being processed
1205        _cfrunLock.acquire()
1206        try:
1207            del _cfrunStatus[host_name]
1208        finally:
1209            _cfrunLock.release()
1210    except:
1211        (type, value, tb) = sys.exc_info()
1212        # Record that this host is no longer being processed
1213        _cfrunLock.acquire()
1214        try:
1215            del _cfrunStatus[host_name]
1216        finally:
1217            _cfrunLock.release()
1218        # Record an error in the log file
1219        if filename != "":
1220            try:
1221                try:
1222                    contents = open(filename, "r").read()
1223                except:
1224                    log_warn("Could not read logfile from cfrun! - %s" % \
1225                            filename, sys.exc_info())
1226                    contents = ""
1227                fp = open(filename, "w")
1228                fp.write("%s\n" % ("*"*80))
1229                fp.write("Command execution failed!\n")
1230                fp.write("Cmdline: %s\n" % cmdline)
1231                fp.write("Error: %s\n" % value)
1232                fp.write("%s\n\n" % ("*"*80))
1233                fp.write(contents)
1234                fp.close()
1235            except: pass
1236        log_error("Failed to initiate cfrun for host: %s" % host_name, \
1237                (type, value, tb))
1238   
1239    # Release the semaphore
1240    _threadLimit.release()
1241    return
1242
1243#####################################################################
1244# Functions to maintain / deal with checked out Cfengine configs
1245#####################################################################
1246@catchEvent("revisionCreated")
1247def updateCfInputDirCB(eventName, host_id, session_id, **kwargs):
1248    """Callback function to trigger updates when revisions are created"""
1249   
1250    autoupdate = int(config_get("cfengine", "autoupdate", "0"))
1251    if autoupdate:
1252        updateCfInputDir(session_id)
1253   
1254@exportViaXMLRPC(SESSION_RW, AUTH_ADMINISTRATOR)
1255def updateCfInputDir(session_id, revNum=-1):
1256    """Updates the cfengine input directory.
1257
1258    Triggered explicitly by user or on the revisionCreated hook.
1259    """
1260   
1261    svn = pysvn.Client()       
1262
1263    # See if there is a checked out revision
1264    gotrev = 0
1265    try:
1266        list = svn.ls(_cfInputDir)
1267        gotrev = 1
1268    except:
1269        # No checked out revision, remove bogus directory
1270        removeDir(_cfInputDir)
1271        ensureDirExists(_cfInputDir)
1272        pass
1273
1274    if revNum == -1:
1275        # Use head revision
1276        revH = pysvn.Revision(pysvn.opt_revision_kind.head)
1277    else:
1278        # Use specified revision
1279        revH = pysvn.Revision(pysvn.opt_revision_kind.number, revNum)
1280   
1281    if gotrev:
1282        # Update the directory
1283        rev = svn.update(_cfInputDir, True, revH)
1284    else:
1285        # Checkout the directory
1286        rev = svn.checkout("%s/inputs" % _svnroot, _cfInputDir, True, revH)
1287   
1288    # Update the revision information files
1289    log_command("%s/update-revisioninfo \"%s\"" % (_cfInputDir, _cfInputDir))
1290
1291    log_info("Updated cfengine configuration to revision %s." % rev.number)
1292
1293#####################################################################
1294# Functions to support the web based configuration browser
1295#####################################################################
1296@exportViaXMLRPC(SESSION_RO, AUTH_ADMINISTRATOR)
1297def viewConfigAtPath(session_id, path, revno=None, order=None, direction=0, \
1298        doMarkup=True):
1299    """Returns the configurations present at the specified path in the repo
1300   
1301    If the path is a directory you get a directory listing back
1302    If the path is a file you get the file contents back along with other
1303    metadata.
1304
1305    If the revno parameter is set the configuration is shown as it existed
1306    in the specified revision. If the revno parameter is None then the
1307    current (HEAD) revision is shown.
1308   
1309    The order parameter may be used to specify a parameter and order to sort
1310    the list that is returned. Valid parameters are name, size and date. Order
1311    may be 1 for descending or 0 for ascending.
1312   
1313    Unless the markup parameter is set to false the contents of the file will
1314    be processed through enscript and HTML will be returned.
1315    """
1316
1317    items = {}
1318   
1319    revision = ccs_revision(checkout=False)
1320    path = path.replace("//","/")
1321   
1322    (revno, entries) = revision.ls(path, revno)
1323    if revno == -1:
1324        # An error occured
1325        raise ccs_cfengine_error("Could not retrieve config at path: %s" % \
1326                path)
1327    for entry in entries:
1328        entry["path"] = entry["name"]
1329        entry["name"] = os.path.basename(entry["name"])
1330        if entry["path"].find("%s/" % entry["name"]) != -1:
1331            # psyvn bug, filename is duplicated
1332            entry["path"] = entry["path"][:-(len(entry["name"])+1)]
1333        if entry["path"].startswith(revision.svnroot):
1334            entry["path"] = entry["path"][len(revision.svnroot)+1:]
1335        rev = {}
1336        rev["date"] = entry["created_rev"].date
1337        rev["kind"] = entry["created_rev"].kind
1338        rev["number"] = entry["created_rev"].number
1339        rev["log"] = revision.getLog(rev["number"])
1340        entry["created_rev"] = rev
1341        entry["age"] = time.strftime("%d/%m/%Y %H:%M:%S", \
1342                time.localtime(entry["time"]))
1343        entry["age_rounded"] = roundTime(time.time() - entry["time"])
1344
1345    # Sort if necessary
1346    if order is not None and order != "":
1347        direction = int(direction)
1348        if order not in ["name", "size", "date"]:
1349            raise ccs_cfengine_error("Invalid order parameter: %s" % order)
1350        if direction != 0 and direction != 1:
1351            raise ccs_cfengine_error("Invalid direction parameter: %s" % \
1352                    direction)
1353        if order == "date": 
1354            order="time"
1355            direction += 1
1356        if direction == 1:
1357            entries.sort(lambda x,y: cmp(x[order], y[order])*-1)
1358        else:
1359            entries.sort(lambda x,y: cmp(x[order], y[order]))
1360
1361    # Return now if there are multiple files or no files
1362    if len(entries) == 0 or len(entries) > 1:
1363        return [revno, entries]
1364    # or a single directory
1365    if entries[0]["kind"] != pysvn.node_kind.file:
1366        return [revno, entries]
1367
1368    # Single file, does it match the path we were given?
1369    if entries[0]["path"] == path:
1370        # Get the contents
1371        file = revision.getFile(path)
1372        props = revision.getProps(path)
1373        entries[0]["props"] = props
1374        if doMarkup:
1375            if "ccs:format" in props.keys():
1376                entries[0]["contents"] = markup(file, props["ccs:format"])
1377            else:
1378                entries[0]["contents"] = markup(file)
1379
1380    return [revno, entries]
1381
1382def markup(file, format="sh"):
1383    """Passes the file through enscript for syntax highlighting"""
1384   
1385    # Generate the command line to pass to enscript
1386    cmdline = config_get("cfengine", "enscript_path", "enscript")
1387    cmdline += ' --color -h -q --language=html -p - -E' + format
1388    log_debug("Enscript command line: %s" % cmdline)
1389   
1390    # Run enscript
1391    (fdi, fdo) = os.popen2(cmdline)
1392    fdi.write(file)
1393    fdi.close()
1394    odata = fdo.read()
1395    rv = fdo.close()
1396    if rv is not None:
1397        log_warn("Could not run enscript to markup file!")
1398        log_debug(odata)
1399        return file.split("\n")
1400
1401    # Strip header and footer
1402    i = odata.find('<PRE>')
1403    beg = i > 0 and i + 6
1404    i = odata.rfind('</PRE>')
1405    end = i > 0 and i or len(odata)
1406    odata = EnscriptDeuglifier().format(odata[beg:end])
1407    return odata.splitlines()
1408
1409# Enscript Deuglifier code from Trac
1410#
1411# Copyright (C) 2003-2006 Edgewall Software
1412# All rights reserved.
1413#
1414# Licensed using the Modified BSD license
1415class EnscriptDeuglifier(object):
1416    def __new__(cls):
1417        self = object.__new__(cls)
1418        if not hasattr(cls, '_compiled_rules'):
1419            cls._compiled_rules = re.compile('(?:'+'|'.join(cls.rules())+')')
1420        self._compiled_rules = cls._compiled_rules
1421        return self
1422   
1423    def format(self, indata):
1424        return re.sub(self._compiled_rules, self.replace, indata)
1425
1426    def replace(self, fullmatch):
1427        for mtype, match in fullmatch.groupdict().items():
1428            if match:
1429                if mtype == 'font':
1430                    return '<span>'
1431                elif mtype == 'endfont':
1432                    return '</span>'
1433                return '<span class="code-%s">' % mtype
1434   
1435    def rules(cls):
1436        return [
1437            r'(?P<comment><FONT COLOR="#B22222">)',
1438            r'(?P<keyword><FONT COLOR="#5F9EA0">)',
1439            r'(?P<type><FONT COLOR="#228B22">)',
1440            r'(?P<string><FONT COLOR="#BC8F8F">)',
1441            r'(?P<func><FONT COLOR="#0000FF">)',
1442            r'(?P<prep><FONT COLOR="#B8860B">)',
1443            r'(?P<lang><FONT COLOR="#A020F0">)',
1444            r'(?P<var><FONT COLOR="#DA70D6">)',
1445            r'(?P<font><FONT.*?>)',
1446            r'(?P<endfont></FONT>)'
1447        ]
1448    rules = classmethod(rules)
1449
1450#####################################################################
1451# Cfengine Resource Server
1452#####################################################################
1453class cfserver(resource.Resource):
1454    """Implements a simple webserver to retrieve resources related to cfengine
1455
1456    Called when the main server receives a query for /cfengine/*
1457
1458    Currently supports the following resources
1459    /cfengine/update.conf/<host_name>
1460        Retrieves an update.conf file for the specified host
1461    /cfkey/keyname
1462        Retrieves the specified cfengine key
1463       
1464    Requests to this resource are restricted to the IP address ranges
1465    specified in the allow_requests_from configuration directive.
1466    """
1467   
1468    resourceName = "CFengine Resource Server"
1469
1470    # Mark as leaf so that render gets called
1471    isLeaf = 1
1472   
1473    def render(self, request):
1474
1475        # XXX: Verify request IP
1476       
1477        log_debug("Received cfengine request for %s" % request.path)
1478       
1479        parts = request.path.split("/")
1480        path = "/".join(parts[2:])
1481       
1482        # Process request
1483        if path.startswith("update.conf"):
1484            return self.getUpdateConf(path, request)
1485        elif path.startswith("cfkey"):
1486            return self.getCfkey(path, request)
1487        elif path.startswith("hostvars"):
1488            return self.getHostvars(path, request)
1489        else:
1490            request.setResponseCode(404, "Unknown cfengine resource requested!")
1491            request.finish()
1492            return server.NOT_DONE_YET
1493
1494    def getCfkey(self, path, request):
1495        """Returns the contents of a cfengine key for the specified host."""
1496       
1497        parts = path.split("/")
1498       
1499        # Get a revision to grab the key from
1500        revision = ccs_revision()
1501        keydir = "%s/ppkeys" % revision.getWorkingDir()
1502       
1503        # Retrieve the requested key
1504        keyfile = "%s/%s" % (keydir, parts[1])
1505
1506        try:
1507            fd = open(keyfile, "r")
1508            content = fd.readlines()
1509            fd.close()
1510        except:
1511            log_warn("Could not open Cfengine key file!", sys.exc_info())
1512            request.setResponseCode(404, "Unknown cfengine key requested!")
1513            request.finish()
1514            return server.NOT_DONE_YET
1515
1516        return "".join(content)
1517
1518    def getUpdateConf(self, path, request):
1519        """Returns the contents of an update.conf file for the specified host.
1520
1521        Currently the host parameter is ignored.
1522        """
1523       
1524        # Get a revision to grab the update.conf from
1525        revision = ccs_revision()
1526        revdir = revision.getConfigBase()
1527       
1528        try:
1529            fd = open("%s/cfconf/update.conf" % revdir, "r")
1530        except:
1531            request.setResponseCode(503, "update.conf unavailable. Please " \
1532                    "try again later")
1533            request.finish()
1534            return server.NOT_DONE_YET
1535       
1536        contents = fd.readlines()
1537        fd.close()
1538       
1539        return "".join(contents)
1540   
1541    def getHostvars(self, path, request):
1542        """Returns the contents of an update.conf file for the specified host.
1543
1544        Currently the host parameter is ignored.
1545        """
1546       
1547        parts = path.split("/")
1548        if len(parts)<2:
1549            request.setResponseCode(404, "Hostname not specified" % host)
1550            request.finish()
1551            return server.NOT_DONE_YET
1552        host = parts[1]
1553       
1554        # Get a revision to grab the hostvars file from
1555        revision = ccs_revision()
1556        revdir = revision.getConfigBase()
1557       
1558        try:
1559            fd = open("%s/hosts/%s/cf.hostvars" % (revdir, host), "r")
1560        except:
1561            request.setResponseCode(404, "%s hostvars not found!" % host)
1562            request.finish()
1563            return server.NOT_DONE_YET
1564       
1565        contents = fd.readlines()
1566        fd.close()
1567       
1568        return "".join(contents)
1569   
1570#####################################################################
1571# Revision Class
1572#####################################################################
1573class ccs_revision:
1574    """Wrapper for a revision of files.
1575   
1576    This class wraps the generation of an entire revision of the configuration
1577    files managed by this system.
1578   
1579    It contains the methods needed to generate the files, insert them into
1580    version control (svn) and then update then pass the resulting revision
1581    identifier back to it's caller.
1582    """
1583
1584    # Location of the svnroot
1585    svnroot = None
1586   
1587    def __init__(self, parentSession=None, changeset=None, checkout=True):
1588        """Creates a ccs_revision class.
1589
1590        If parentSession or changeset are not specified or None, a read-only
1591        revision is created. This is only useful if you want to inspect the
1592        repository without making any changes.
1593        """
1594       
1595        self.mParentSession = parentSession
1596        self.mChangeset = changeset
1597        self.checkout = checkout
1598        self.pendingProps = {}
1599        self.lock = threading.RLock()
1600       
1601        # Get a SVN client to use for this revision
1602        self.mSvn = pysvn.Client()
1603       
1604        if self.checkout:
1605            # Setup a working directory for this revision
1606            self.rDir = mkdtemp("", "ccsd")
1607       
1608            # Checkout the current configuration HEAD to this directory
1609            rev = self.mSvn.checkout(self.svnroot, self.rDir)
1610            self.mCurRev = rev
1611       
1612            # Check basic repository structure
1613            if self.mParentSession is not None and self.mChangeset is not None:
1614                self.checkRepoStructure()
1615        else:
1616            self.rDir = None
1617            self.mCurRev = None
1618           
1619        # Start with no errors
1620        self.mErrors = {}
1621       
1622    def __del__(self):
1623       
1624        # Nothing to check if nothing was checked out
1625        if self.rDir is None:
1626            return
1627
1628        # Don't check the status of a read-only revision
1629        if self.mParentSession is None or self.mChangeset is None:
1630            removeDir(self.rDir)
1631            return
1632       
1633        self.lock.acquire()
1634        try:
1635            # Check status of working directory
1636            try:
1637                status = self.mSvn.status(self.rDir, True, True)
1638            except pysvn.ClientError:
1639                # Probably never got checked out
1640                status = []           
1641        finally:
1642            self.lock.release()
1643           
1644        flag=0
1645        for entry in status:
1646            if entry.text_status == pysvn.wc_status_kind.ignored:
1647                continue
1648            if entry.text_status != pysvn.wc_status_kind.normal:
1649                log_debug("File %s modified (%s) in changeset %s" % \
1650                        (entry.path, entry.text_status, self.mChangeset))
1651                flag=1
1652       
1653        if flag==1:
1654            log_warn("Revision object for changeset %s in " \
1655                    "session %s deleted before checkin. Changes lost!" % \
1656                    (self.mChangeset, self.mParentSession.session_id))
1657               
1658        # Clean up the working directory
1659        removeDir(self.rDir)
1660       
1661    def getWorkingDir(self):
1662        """Returns the path that the repository is checked out into"""
1663
1664        return self.rDir
1665   
1666    def getConfigBase(self):
1667        """Returns the path that cfengine config files should live in"""
1668        if self.rDir is None:
1669            return None
1670        return "%s/inputs" % self.rDir
1671   
1672    def _checkForModified(self, cDir):
1673        """Performs svn actions on changed files in the specified directory"""
1674       
1675        # Nothing to check if no repository is checked out
1676        if self.rDir is None:
1677            return None
1678       
1679        # Don't check the status of a read-only revision
1680        if self.mParentSession is None or self.mChangeset is None:
1681            log_warn("Cannot check status on a read-only revision!")
1682            return
1683       
1684        # Recurse through if we were passed a list
1685        if type(cDir) == type([]):
1686            for d in cDir:
1687                self._checkForModified(d)
1688            return
1689
1690        self.lock.acquire()
1691        try:
1692            # Check state
1693            status = self.mSvn.status(cDir, True, True)
1694            for entry in status:
1695                # Ensure that ccs-revision files are always ignored
1696                if os.path.isdir(entry.path):
1697                    self.propset(entry.path, "svn:ignore", "ccs-revision")
1698                if entry.text_status == pysvn.wc_status_kind.unversioned:
1699                    if entry.path.startswith(self.rDir):
1700                        ename = entry.path[len(self.rDir)+1:]
1701                    log_debug("Added %s in changeset %s" % \
1702                            (ename, self.mChangeset))
1703                    self.mSvn.add(entry.path, False)
1704                    # Recurse if we added a directory
1705                    if os.path.isdir(entry.path):
1706                        self._checkForModified(entry.path)
1707                    else:
1708                        # Ensure the date property is set on files
1709                        self.propset(entry.path, "svn:keywords", "Date")
1710                        # Check if there are any other pending properties to
1711                        # set
1712                        path = entry.path
1713                        if path in self.pendingProps.keys():
1714                            for prop,value in self.pendingProps[path].items():
1715                                self.propset(path, prop, value, False)
1716                elif entry.text_status == pysvn.wc_status_kind.modified or \
1717                        entry.text_status == pysvn.wc_status_kind.added or \
1718                        entry.text_status == pysvn.wc_status_kind.ignored or \
1719                        entry.text_status == pysvn.wc_status_kind.deleted:
1720                    # Don't need to mention modified files that are already
1721                    # processed
1722                    pass
1723                elif entry.text_status != pysvn.wc_status_kind.normal:
1724                    log_debug("%s (%s) has bad state in changeset %s!" % \
1725                            (entry.path, entry.text_status, self.mChangeset))
1726        finally:
1727            self.lock.release()
1728       
1729    def propset(self, path, prop, value, canDefer=True):
1730        """Sets the specified property on the specified path"""
1731       
1732        if self.mParentSession is None or self.mChangeset is None:
1733            raise ccs_revision_error("Cannot set property on read-only " \
1734                    "revision")
1735       
1736        # Use the svn repo directly if nothing is checked out
1737        if self.rDir is None:
1738            base = self.svnroot
1739        else:
1740            base = self.rDir
1741               
1742        try:
1743            if not path.startswith(base):
1744                filename = "%s/%s" % (base, path)
1745            else:
1746                filename = path
1747            self.lock.acquire()
1748            try:
1749                rev = self.mSvn.propset(prop, value, filename)
1750            finally:
1751                self.lock.release()
1752            if self.rDir is None:
1753                self.saveRevProps(rev)
1754        except pysvn.ClientError:
1755            # Add to a list of pending properties that we'll try and set on
1756            # commit after adding the file. This covers the case where a
1757            # caller tries to set a property on an as yet unversioned file
1758            if filename in self.pendingProps.keys():
1759                self.pendingProps[filename][prop] = value
1760            else:
1761                self.pendingProps[filename] = {prop:value}
1762            log_debug("Deferred propset on %s" % filename, sys.exc_info())
1763        except:
1764            log_error("Could not set %s to %s on %s" % \
1765                    (prop, value, filename), sys.exc_info())
1766
1767    @registerEvent("revisionCreated")
1768    def checkin(self, message, paths=None):
1769        """Checks in the changes to the repository with the specified message"""
1770       
1771        if self.rDir is None:
1772            raise ccs_revision_error("Cannot checkin. Not checked out!")
1773       
1774        if self.mParentSession is None or self.mChangeset is None:
1775            raise ccs_revision_error("Cannot checkin a read-only revision")
1776       
1777        # Default to the whole repository
1778        if paths is None:
1779            paths = self.rDir
1780           
1781        # Check status of working directory and add / del files etc
1782        self._checkForModified(paths)
1783       
1784        self.lock.acquire()
1785        try:
1786            r = self.mSvn.checkin(paths, message, True)
1787            n = self.saveRevProps(r)
1788        finally:
1789            self.lock.release()
1790        if n<0:
1791            # Nothing changed
1792            return -1
1793       
1794        triggerEvent(self.mParentSession.session_id, "revisionCreated", \
1795                revision_no=n)
1796
1797        return n
1798
1799    def saveRevProps(self, r):
1800        """Saves customised properties against each revision
1801
1802        These properties are used to help keep track of how the database/svn
1803        repository changes match up.
1804        """
1805       
1806        if r is None:
1807            # Nothing changed
1808            return -1
1809
1810        # Set the author property on the checkin
1811        try:
1812            self.lock.acquire()
1813            try:
1814                r2 = self.mSvn.revpropset("svn:author", \
1815                        self.mParentSession.username, self.svnroot, r)
1816            finally:
1817                self.lock.release()
1818        except:
1819            (type, value, tb) = sys.exc_info()
1820            log_warn("Could not set author property on revision %s - %s" % \
1821                    (r.number, value))
1822            r2 = r
1823       
1824        # Record the changeset that triggered this revision
1825        try:
1826            self.lock.acquire()
1827            try:
1828                r3 = self.mSvn.revpropset("ccs:changeset", "%s" % \
1829                        self.mChangeset, self.svnroot, r2)
1830            finally:
1831                self.lock.release()
1832        except:
1833            (type, value, tb) = sys.exc_info()
1834            log_warn("Could not set changeset property on " \
1835                    "revision %s - %s" % (r2.number, value))
1836
1837        log_info("Committed revision %s to version control" % r2.number)
1838        return r2.number
1839   
1840    def fileExists(self, path):
1841        """Checks if the specified file exists in the repository"""
1842       
1843        # Use the svn repo directly if nothing is checked out
1844        if self.rDir is None:
1845            base = self.svnroot
1846        else:
1847            base = self.rDir
1848               
1849        try:
1850            if not path.startswith(base):
1851                filename = "%s/%s" % (base, path)
1852            else:
1853                filename = path
1854            self.lock.acquire()
1855            try:
1856                e = self.mSvn.ls(filename)
1857            finally:
1858                self.lock.release()
1859            if len(e) > 0:
1860                return True
1861        except:
1862            log_warn("Failed to check existance of file: %s" % filename)
1863            return False
1864       
1865        return False
1866   
1867    def ls(self, path, revno=None):
1868        """Returns a list of entries in the specified directory"""
1869       
1870        # Use the svn repo directly if nothing is checked out
1871        if self.rDir is None:
1872            base = self.svnroot
1873        else:
1874            base = self.rDir
1875               
1876        try:
1877            if not path.startswith(base):
1878                filename = "%s/%s" % (base, path)
1879            else:
1880                filename = path
1881               
1882            self.lock.acquire()
1883            try:
1884                if revno is None or revno=="":
1885                    rev = pysvn.Revision(pysvn.opt_revision_kind.head)
1886                else:
1887                    rev = pysvn.Revision(pysvn.opt_revision_kind.number, revno)
1888                e = self.mSvn.ls(filename.rstrip("/"), rev)
1889            finally:
1890                self.lock.release()
1891            if len(e) > 0:
1892                if revno is None:
1893                    revno = self.getYoungestRevision()
1894                return (revno, e)
1895        except:
1896            log_warn("Could not list directory: %s" % filename, sys.exc_info())
1897       
1898        return (-1, [])
1899       
1900    def getYoungestRevision(self, path=""):
1901        """Returns the number of the youngest revision in the repository"""
1902        revno = -1
1903
1904        if path != "":
1905            path = "%s/%s" % (self.svnroot, path)
1906        else:
1907            path = self.svnroot
1908       
1909        try:
1910            self.lock.acquire()
1911            t = mkdtemp("", "ccsd")
1912            try:
1913                l = self.mSvn.log(path)
1914                for entry in l:
1915                    # Discard revisions which touched only the directory
1916                    r1 = pysvn.Revision(pysvn.opt_revision_kind.number, \
1917                            entry["revision"].number-1)
1918                    r2 = pysvn.Revision(pysvn.opt_revision_kind.number, \
1919                            entry["revision"].number)
1920                    try:
1921                        diff = self.mSvn.diff(t, path, r1, r2).strip()
1922                    except:
1923                        (type, value, tb) = sys.exc_info()
1924                        if str(value).find("was not found") != -1:
1925                            # Earlier revision not found, must be added
1926                            revno = entry["revision"].number
1927                            break
1928                    ignore = True
1929                    for line in diff.split("\n"):
1930                        if line.strip() == "Property changes on:":
1931                            continue
1932                        if line.startswith("Property changes on:"):
1933                            ignore = False
1934                            break
1935                        if line.startswith("Index:"):
1936                            ignore = False
1937                            break
1938                    if ignore:
1939                        continue
1940                    # Found a revision which touched inside the directory
1941                    revno = entry["revision"].number
1942                    break
1943            finally:
1944                self.lock.release()
1945                removeDir(t)
1946        except:
1947            log_warn("Could not determine youngest revision", sys.exc_info())
1948
1949        return revno
1950   
1951    def getLog(self, revno):
1952        """Returns the log message for the specified revision"""
1953
1954        try:
1955            self.lock.acquire()
1956            try:
1957                rev = pysvn.Revision(pysvn.opt_revision_kind.number, revno)
1958                info = self.mSvn.revpropget("svn:log", self.svnroot, rev)
1959            finally:
1960                self.lock.release()
1961            return info[1].strip()
1962        except:
1963            log_warn("Could not retrieve log for revision %s" % revno, \
1964                    sys.exc_info())
1965
1966        return ""
1967   
1968    def getFile(self, path, revno=None):
1969        """Returns the contents of the file at the specified path and rev"""
1970       
1971        # Use the svn repo directly if nothing is checked out
1972        if self.rDir is None:
1973            base = self.svnroot
1974        else:
1975            base = self.rDir
1976               
1977        try:
1978            if not path.startswith(base):
1979                filename = "%s/%s" % (base, path)
1980            else:
1981                filename = path
1982            self.lock.acquire()
1983            try:
1984                if revno is None:
1985                    rev = pysvn.Revision(pysvn.opt_revision_kind.head)
1986                else:
1987                    rev = pysvn.Revision(pysvn.opt_revision_kind.number, revno)
1988                contents = self.mSvn.cat(filename.rstrip("/"), rev)
1989            finally:
1990                self.lock.release()
1991            return contents
1992        except:
1993            log_warn("Could not retrieve file: %s" % filename, sys.exc_info())
1994       
1995        return ""
1996
1997    def getProps(self, path, revno=None):
1998        """Returns the properties set on the specified file"""
1999       
2000        # Use the svn repo directly if nothing is checked out
2001        if self.rDir is None:
2002            base = self.svnroot
2003        else:
2004            base = self.rDir
2005               
2006        try:
2007            if not path.startswith(base):
2008                filename = "%s/%s" % (base, path)
2009            else:
2010                filename = path
2011            self.lock.acquire()
2012            try:
2013                if revno is None:
2014                    rev = pysvn.Revision(pysvn.opt_revision_kind.head)
2015                else:
2016                    rev = pysvn.Revision(pysvn.opt_revision_kind.number, revno)
2017                contents = self.mSvn.proplist(filename.rstrip("/"), rev)
2018            finally:
2019                self.lock.release()
2020            if len(contents) > 0:
2021                return contents[0][1]
2022        except:
2023            log_warn("Could not retrieve properties: %s" % \
2024                    filename, sys.exc_info())
2025       
2026        return ""
2027
2028    def checkRepoStructure(self):
2029        """Checks the repository has all the required base directories.
2030
2031        If a base directory is not present it is created. The base directories
2032        are:
2033        inputs/        Host/Service configuration files
2034
2035        Further hierarchy within each base directory is the responsibility of
2036        other modules.
2037        """
2038       
2039        if self.rDir is None:
2040            raise ccs_revision_error("Cannot check repository structure. " \
2041                    "Not checked out!")
2042       
2043        if self.mParentSession is None or self.mChangeset is None:
2044            log_warn("Cannot check repository structure on a read-only " \
2045                    "revision")
2046            return
2047       
2048        flag = 0
2049       
2050        configsDir = self.getConfigBase()
2051        n = ensureDirExists(configsDir)
2052        self.lock.acquire()
2053        try:
2054            if n > 0:
2055                # Schedule Additions
2056                bDir = configsDir
2057                while n>1:
2058                    bDir = os.path.dirname(configsDir)
2059                    n-=1
2060                log_info("Created configuration directory (%s) within " \
2061                        "repository" % configsDir)
2062                self.mSvn.add(bDir, True)
2063                flag = 1
2064           
2065            # Commit directory changes immediately
2066            if flag==1:
2067                r = self.mSvn.checkin(self.rDir, \
2068                        "checkRepoStructure created missing directories", True)
2069        finally:
2070            self.lock.release()
2071
2072        # Check for the script that updates the ccs-revision files
2073        filename = "%s/update-revisioninfo" % configsDir
2074        docreate=False
2075        doupdate=False
2076        if not os.path.exists(filename):
2077            docreate=True
2078        else:
2079            contents = file(filename, "r").read()
2080            if contents.find("svn info") == -1:
2081                doupdate = True
2082        if docreate or doupdate:
2083            self.lock.acquire()
2084            try:
2085                # Create the script
2086                fp = open(filename, "w")
2087                fp.write("""#!/bin/bash
2088echo "Updating version information"
2089for d in `find $1 -type d | grep -v ".svn" | xargs`; do
2090    version=`svn info $d/* | grep "Last Changed Rev" | awk '{print $4}' | sort -nr | head -n1`
2091    echo "  $d -> $version"
2092    echo "$version" > $d/ccs-revision
2093done
2094# vim:set sw=4 ts=4 sts=4 et:
2095""")
2096                fp.close()
2097                if docreate:
2098                    self.mSvn.add(filename, False)
2099                    self.mSvn.propset("svn:executable", "true", filename)
2100                    action = "Created"
2101                else:
2102                    action = "Updated"
2103                self.mSvn.checkin(filename, \
2104                        "%s update-revisioninfo script" % action)
2105            finally:
2106                self.lock.release()
2107
2108def validateRepository(svnroot):
2109    """Checks for the existance of a valid svn repository at svnroot"""
2110
2111    # Extract the path from the URL
2112    svnpath = svnroot[svnroot.find("://")+3:]
2113
2114    # Check directory exists
2115    if not os.path.exists(svnpath):
2116        log_debug("Specified repository path does not exist: %s" % svnpath)
2117        return False
2118   
2119    # Check it looks a bit like a subversion repository
2120    fp = open("%s/README.txt" % svnpath, "r")
2121    if not fp:
2122        log_debug("No README.txt inside repository: %s" % svnpath)
2123        return False
2124    tmp = fp.read().strip()
2125    fp.close()
2126    if not tmp.startswith("This is a Subversion repository"):
2127        log_debug("invalid README.txt in repository: %s" % svnpath)       
2128        return False
2129
2130    # Final, master check, can we check it out, instantiate a revision class
2131    revision = ccs_revision(True, True)
2132    del revision
2133
2134    # All OK
2135    return True
2136
2137def createRepository(svnroot):
2138    """Initialises a blank repository and configures it for use by ccsd"""
2139
2140    # Look for the svnadmin utility
2141    fd = os.popen("/usr/bin/which svnadmin 2>/dev/null")
2142    path = fd.read().strip()
2143    if fd.close() != None:
2144        log_error("Cannot find svnadmin utility.")
2145        return False
2146   
2147    # Check we can execute the svnadmin utilty
2148    if not os.access(path, os.X_OK):
2149        log_error("Cannot execute svnadmin utility.")
2150        return False
2151
2152    # Try and create the repository
2153    svnpath = svnroot[svnroot.find("://")+3:]
2154    fd = os.popen("%s create --fs-type fsfs %s 2>&1" % (path, svnpath))
2155    result = fd.read().strip()
2156    if fd.close() != None:
2157        log_error("svnadmin create failed. See tb log for more details.")
2158        log_tb(None, result)
2159        return False
2160   
2161    # Add a pre-revprop-change hook to allow the daemon to set author
2162    # revision properties
2163    hook = "%s/hooks/pre-revprop-change" % svnpath
2164    fd = open(hook, "w")
2165    if not fd:
2166        log_warning("Unable to setup pre-revprop-change hook in new " \
2167                "repository!")
2168    else:
2169        fd.write("""#!/bin/bash
2170
2171# PRE-REVPROP-CHANGE HOOK
2172
2173# Created by the CRCnet Configuration System Daemon on
2174# %s
2175
2176# Allow all changes at this time
2177exit 0
2178""" % time.ctime())
2179        fd.close()
2180        os.chmod(hook, 0755)
2181
2182    # Success
2183    log_info("Subversion repository created at %s" % svnpath)
2184    return True
2185
2186#####################################################################
2187# Cfengine Integration Initialisation
2188#####################################################################
2189# Base CFengine paths
2190_cfBaseDir = ""
2191_cfInputDir = ""
2192_svnroot = ""
2193# Global information about template generation
2194_hostTemplates = {}
2195_networkTemplates = {}
2196_templateDir = ""
2197_templateModDir = ""
2198_events = {}
2199_templateStats = {}
2200_statsLock = None
2201# Global information about cfrun instances
2202_cfrunStatus = {}
2203_cfrunLock = None
2204# Semaphore to control how many threads we initiate
2205_threadLimit = None
2206
2207def initCFengine():
2208    global _hostTemplates, _networkTemplates, _templateDir, _templateModDir
2209    global _events, _cfBaseDir, _cfInputDir, _svnroot, _statsLock, _threadLimit
2210    global _cfrunLock
2211
2212    # Retrieve template directory configuration
2213    _templateDir = config_get("cfengine", "template_dir")
2214    if _templateDir == None or _templateDir == "":
2215        log_fatal("Invalid template directory (%s)!" % _templateDir)
2216    _templateModDir = config_get("cfengine", "template_module_dir")
2217    if _templateModDir == None or _templateModDir == "":
2218        log_fatal("Invalid template module directory (%s)!" % _templateModDir)
2219
2220    # Retrieve cfengine configuration file directory configuration
2221    _cfBaseDir = config_get("cfengine", "cfbase_dir")
2222    if _cfBaseDir == None or _cfBaseDir == "":
2223        log_fatal("Invalid cfengine base directory (%s)!" % _cfBaseDir)
2224    _cfInputDir = "%s/inputs" % _cfBaseDir
2225
2226    # Decide where the svn root is
2227    _svnroot = config_get("cfengine", "config_svnroot")     
2228    if _svnroot == None or _svnroot == "":
2229        log_fatal("Configuration Data subversion repository not specified!")
2230    # Check that the svnroot is a valid svn URL
2231    if _svnroot.find("://")==-1:
2232        log_fatal("Invalid svnroot URL: %s" % _svnroot)   
2233    # Let the revision class know where to get its data from
2234    ccs_revision.svnroot = _svnroot
2235    # Validate the repository and create it if it doesn't exist
2236    if not validateRepository(_svnroot):
2237        log_warn("No svn repository at specified path. Attempting " \
2238                "to create one.")
2239        if not createRepository(_svnroot):
2240            log_fatal("Could not create repository at %s" % _svnroot)
2241   
2242    # Initialise the statistics lock
2243    _statsLock = threading.Lock()
2244
2245    # Initialise the cfrun lock
2246    _cfrunLock = threading.Lock()
2247   
2248    # How many threads are we allowed to run at a time
2249    maxThreads = config_get("cfengine", "max_threads", DEFAULT_MAX_THREADS)
2250    suggestThreadpoolSize(maxThreads)
2251    _threadLimit = threading.Semaphore(maxThreads)
2252   
2253    # Initialise the templates
2254    initTemplates()
2255   
2256    # Register Cfengine Resource Server
2257    registerResource("cfengine", cfserver)
Note: See TracBrowser for help on using the repository browser.