#!/usr/bin/perl #------------------------------------------------------------------------------ # Free realtime web server logfile analyzer to show advanced web statistics. # Works from command line or as a CGI. You must use this script as often as # necessary from your scheduler to update your statistics and from command # line or a browser to read report results. # See AWStats documentation (in docs/ directory) for all setup instructions. #------------------------------------------------------------------------------ # $Revision: 1.814 $ - $Author: eldy $ - $Date: 2005/02/24 21:02:23 $ require 5.005; #$|=1; #use warnings; # Must be used in test mode only. This reduce a little process speed #use diagnostics; # Must be used in test mode only. This reduce a lot of process speed use strict;no strict "refs"; use Time::Local; # use Time::Local 'timelocal_nocheck' is faster but not supported by all Time::Local modules use Socket; use Jcode; #------------------------------------------------------------------------------ # Defines #------------------------------------------------------------------------------ use vars qw/ $REVISION $VERSION /; $REVISION='$Revision: 1.814 $'; $REVISION =~ /\s(.*)\s/; $REVISION=$1; $VERSION="6.4 (build $REVISION)"; # ----- Constants ----- use vars qw/ $DEBUGFORCED $NBOFLINESFORBENCHMARK $FRAMEWIDTH $NBOFLASTUPDATELOOKUPTOSAVE $LIMITFLUSH $NEWDAYVISITTIMEOUT $VISITTIMEOUT $NOTSORTEDRECORDTOLERANCE $WIDTHCOLICON $TOOLTIPON $lastyearbeforeupdate /; $DEBUGFORCED=0; # Force debug level to log lesser level into debug.log file (Keep this value to 0) $NBOFLINESFORBENCHMARK=8192; # Benchmark info are printing every NBOFLINESFORBENCHMARK lines (Must be a power of 2) $FRAMEWIDTH=240; # Width of left frame when UseFramesWhenCGI is on $NBOFLASTUPDATELOOKUPTOSAVE=500; # Nb of records to save in DNS last update cache file $LIMITFLUSH=5000; # Nb of records in data arrays after how we need to flush data on disk $NEWDAYVISITTIMEOUT=764041; # Delay between 01-23:59:59 and 02-00:00:00 $VISITTIMEOUT=10000; # Lapse of time to consider a page load as a new visit. 10000 = 1 hour (Default = 10000) $NOTSORTEDRECORDTOLERANCE=20000; # Lapse of time to accept a record if not in correct order. 20000 = 2 hour (Default = 20000) $WIDTHCOLICON=32; $TOOLTIPON=0; # Tooltips plugin loaded # ----- Running variables ----- use vars qw/ $DIR $PROG $Extension $Debug $ShowSteps $DebugResetDone $DNSLookupAlreadyDone $RunAsCli $UpdateFor $HeaderHTTPSent $HeaderHTMLSent $LastLine $LastLineNumber $LastLineOffset $LastLineChecksum $LastUpdate $lowerval $PluginMode $TotalUnique $TotalVisits $TotalHostsKnown $TotalHostsUnknown $TotalPages $TotalHits $TotalBytes $TotalNotViewedPages $TotalNotViewedHits $TotalNotViewedBytes $TotalEntries $TotalExits $TotalBytesPages $TotalDifferentPages $TotalKeyphrases $TotalKeywords $TotalDifferentKeyphrases $TotalDifferentKeywords $TotalSearchEnginesPages $TotalSearchEnginesHits $TotalRefererPages $TotalRefererHits $TotalDifferentSearchEngines $TotalDifferentReferer $FrameName $Center $FileConfig $FileSuffix $Host $DayRequired $MonthRequired $YearRequired $QueryString $SiteConfig $StaticLinks $PageCode $PageDir $PerlParsingFormat $UserAgent $pos_vh $pos_host $pos_logname $pos_date $pos_tz $pos_method $pos_url $pos_code $pos_size $pos_referer $pos_agent $pos_query $pos_gzipin $pos_gzipout $pos_compratio $pos_timetaken $pos_cluster $pos_emails $pos_emailr $pos_hostr @pos_extra /; $DIR=$PROG=$Extension=''; $Debug = $ShowSteps = 0; $DebugResetDone = $DNSLookupAlreadyDone = 0; $RunAsCli = $UpdateFor = $HeaderHTTPSent = $HeaderHTMLSent = 0; $LastLine = $LastLineNumber = $LastLineOffset = $LastLineChecksum = $LastUpdate = 0; $lowerval = 0; $PluginMode = ''; $TotalUnique = $TotalVisits = $TotalHostsKnown = $TotalHostsUnknown = 0; $TotalPages = $TotalHits = $TotalBytes = 0; $TotalNotViewedPages = $TotalNotViewedHits = $TotalNotViewedBytes = 0; $TotalEntries = $TotalExits = $TotalBytesPages = $TotalDifferentPages = 0; $TotalKeyphrases = $TotalKeywords = $TotalDifferentKeyphrases = $TotalDifferentKeywords = 0; $TotalSearchEnginesPages = $TotalSearchEnginesHits = $TotalRefererPages = $TotalRefererHits = $TotalDifferentSearchEngines = $TotalDifferentReferer = 0; ($FrameName, $Center, $FileConfig, $FileSuffix, $Host, $DayRequired, $MonthRequired, $YearRequired, $QueryString, $SiteConfig, $StaticLinks, $PageCode, $PageDir, $PerlParsingFormat, $UserAgent)= ('','','','','','','','','','','','','','',''); # ----- Plugins variable ----- use vars qw/ %PluginsLoaded $PluginDir $AtLeastOneSectionPlugin /; %PluginsLoaded=(); $PluginDir=''; $AtLeastOneSectionPlugin=0; # ----- Time vars ----- use vars qw/ $starttime $nowtime $tomorrowtime $nowweekofmonth $nowweekofyear $nowdaymod $nowsmallyear $nowsec $nowmin $nowhour $nowday $nowmonth $nowyear $nowwday $nowyday $nowns $StartSeconds $StartMicroseconds /; $StartSeconds=$StartMicroseconds=0; # ----- Variables for config file reading ----- use vars qw/ $FoundNotPageList /; $FoundNotPageList=0; # ----- Config file variables ----- use vars qw/ $StaticExt $DNSStaticCacheFile $DNSLastUpdateCacheFile $MiscTrackerUrl $Lang $MaxRowsInHTMLOutput $MaxLengthOfShownURL $MaxLengthOfStoredURL $MaxLengthOfStoredUA %BarPng $BuildReportFormat $BuildHistoryFormat $ExtraTrackedRowsLimit /; $StaticExt='html'; $DNSStaticCacheFile='dnscache.txt'; $DNSLastUpdateCacheFile='dnscachelastupdate.txt'; $MiscTrackerUrl='/js/awstats_misc_tracker.js'; $Lang='auto'; $MaxRowsInHTMLOutput=1000; $MaxLengthOfShownURL=64; $MaxLengthOfStoredURL=256; # Note: Apache LimitRequestLine is default to 8190 $MaxLengthOfStoredUA=256; %BarPng=('vv'=>'vv.png','vu'=>'vu.png','hu'=>'hu.png','vp'=>'vp.png','hp'=>'hp.png', 'he'=>'he.png','hx'=>'hx.png','vh'=>'vh.png','hh'=>'hh.png','vk'=>'vk.png','hk'=>'hk.png'); $BuildReportFormat='html'; $BuildHistoryFormat='text'; $ExtraTrackedRowsLimit=500; use vars qw/ $DebugMessages $AllowToUpdateStatsFromBrowser $EnableLockForUpdate $DNSLookup $AllowAccessFromWebToAuthenticatedUsersOnly $BarHeight $BarWidth $CreateDirDataIfNotExists $KeepBackupOfHistoricFiles $NbOfLinesParsed $NbOfLinesDropped $NbOfLinesCorrupted $NbOfOldLines $NbOfNewLines $NbOfLinesShowsteps $NewLinePhase $NbOfLinesForCorruptedLog $PurgeLogFile $ArchiveLogRecords $ShowDropped $ShowCorrupted $ShowUnknownOrigin $ShowLinksToWhoIs $ShowAuthenticatedUsers $ShowFileSizesStats $ShowScreenSizeStats $ShowSMTPErrorsStats $ShowEMailSenders $ShowEMailReceivers $ShowWormsStats $ShowClusterStats $IncludeInternalLinksInOriginSection $AuthenticatedUsersNotCaseSensitive $Expires $UpdateStats $MigrateStats $URLNotCaseSensitive $URLWithQuery $URLReferrerWithQuery $DecodeUA /; ($DebugMessages, $AllowToUpdateStatsFromBrowser, $EnableLockForUpdate, $DNSLookup, $AllowAccessFromWebToAuthenticatedUsersOnly, $BarHeight, $BarWidth, $CreateDirDataIfNotExists, $KeepBackupOfHistoricFiles, $NbOfLinesParsed, $NbOfLinesDropped, $NbOfLinesCorrupted, $NbOfOldLines, $NbOfNewLines, $NbOfLinesShowsteps, $NewLinePhase, $NbOfLinesForCorruptedLog, $PurgeLogFile, $ArchiveLogRecords, $ShowDropped, $ShowCorrupted, $ShowUnknownOrigin, $ShowLinksToWhoIs, $ShowAuthenticatedUsers, $ShowFileSizesStats, $ShowScreenSizeStats, $ShowSMTPErrorsStats, $ShowEMailSenders, $ShowEMailReceivers, $ShowWormsStats, $ShowClusterStats, $IncludeInternalLinksInOriginSection, $AuthenticatedUsersNotCaseSensitive, $Expires, $UpdateStats, $MigrateStats, $URLNotCaseSensitive, $URLWithQuery, $URLReferrerWithQuery, $DecodeUA)= (0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0); use vars qw/ $DetailedReportsOnNewWindows $FirstDayOfWeek $KeyWordsNotSensitive $SaveDatabaseFilesWithPermissionsForEveryone $WarningMessages $ShowLinksOnUrl $UseFramesWhenCGI $ShowMenu $ShowSummary $ShowMonthStats $ShowDaysOfMonthStats $ShowDaysOfWeekStats $ShowHoursStats $ShowDomainsStats $ShowHostsStats $ShowRobotsStats $ShowSessionsStats $ShowPagesStats $ShowFileTypesStats $ShowOSStats $ShowBrowsersStats $ShowOriginStats $ShowKeyphrasesStats $ShowKeywordsStats $ShowMiscStats $ShowHTTPErrorsStats $AddDataArrayMonthStats $AddDataArrayShowDaysOfMonthStats $AddDataArrayShowDaysOfWeekStats $AddDataArrayShowHoursStats /; ($DetailedReportsOnNewWindows, $FirstDayOfWeek, $KeyWordsNotSensitive, $SaveDatabaseFilesWithPermissionsForEveryone, $WarningMessages, $ShowLinksOnUrl, $UseFramesWhenCGI, $ShowMenu, $ShowSummary, $ShowMonthStats, $ShowDaysOfMonthStats, $ShowDaysOfWeekStats, $ShowHoursStats, $ShowDomainsStats, $ShowHostsStats, $ShowRobotsStats, $ShowSessionsStats, $ShowPagesStats, $ShowFileTypesStats, $ShowOSStats, $ShowBrowsersStats, $ShowOriginStats, $ShowKeyphrasesStats, $ShowKeywordsStats, $ShowMiscStats, $ShowHTTPErrorsStats, $AddDataArrayMonthStats, $AddDataArrayShowDaysOfMonthStats, $AddDataArrayShowDaysOfWeekStats, $AddDataArrayShowHoursStats )= (1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1); use vars qw/ $AllowFullYearView $LevelForRobotsDetection $LevelForWormsDetection $LevelForBrowsersDetection $LevelForOSDetection $LevelForRefererAnalyze $LevelForFileTypesDetection $LevelForSearchEnginesDetection $LevelForKeywordsDetection /; ($AllowFullYearView, $LevelForRobotsDetection, $LevelForWormsDetection, $LevelForBrowsersDetection, $LevelForOSDetection, $LevelForRefererAnalyze, $LevelForFileTypesDetection, $LevelForSearchEnginesDetection, $LevelForKeywordsDetection)= (2,2,0,2,2,2,2,2,2); use vars qw/ $DirLock $DirCgi $DirConfig $DirData $DirIcons $DirLang $AWScript $ArchiveFileName $AllowAccessFromWebToFollowingIPAddresses $HTMLHeadSection $HTMLEndSection $LinksToWhoIs $LinksToIPWhoIs $LogFile $LogType $LogFormat $LogSeparator $Logo $LogoLink $StyleSheet $WrapperScript $SiteDomain $UseHTTPSLinkForUrl $URLQuerySeparators $URLWithAnchor $ErrorMessages $ShowFlagLinks /; ($DirLock, $DirCgi, $DirConfig, $DirData, $DirIcons, $DirLang, $AWScript, $ArchiveFileName, $AllowAccessFromWebToFollowingIPAddresses, $HTMLHeadSection, $HTMLEndSection, $LinksToWhoIs, $LinksToIPWhoIs, $LogFile, $LogType, $LogFormat, $LogSeparator, $Logo, $LogoLink, $StyleSheet, $WrapperScript, $SiteDomain, $UseHTTPSLinkForUrl, $URLQuerySeparators, $URLWithAnchor, $ErrorMessages, $ShowFlagLinks)= ('','','','','','','','','','','','','','','','','','','','','','','','','','',''); use vars qw/ $color_Background $color_TableBG $color_TableBGRowTitle $color_TableBGTitle $color_TableBorder $color_TableRowTitle $color_TableTitle $color_text $color_textpercent $color_titletext $color_weekend $color_link $color_hover $color_other $color_h $color_k $color_p $color_e $color_x $color_s $color_u $color_v /; ($color_Background, $color_TableBG, $color_TableBGRowTitle, $color_TableBGTitle, $color_TableBorder, $color_TableRowTitle, $color_TableTitle, $color_text, $color_textpercent, $color_titletext, $color_weekend, $color_link, $color_hover, $color_other, $color_h, $color_k, $color_p, $color_e, $color_x, $color_s, $color_u, $color_v)= ('','','','','','','','','','','','','','','','','','','','','',''); # ---------- Init arrays -------- use vars qw/ @RobotsSearchIDOrder_list1 @RobotsSearchIDOrder_list2 @RobotsSearchIDOrder_listgen @SearchEnginesSearchIDOrder_list1 @SearchEnginesSearchIDOrder_list2 @SearchEnginesSearchIDOrder_listgen @BrowsersSearchIDOrder @OSSearchIDOrder @WordsToExtractSearchUrl @WordsToCleanSearchUrl @WormsSearchIDOrder @RobotsSearchIDOrder @SearchEnginesSearchIDOrder @_from_p @_from_h @_time_p @_time_h @_time_k @_time_nv_p @_time_nv_h @_time_nv_k @DOWIndex @fieldlib @keylist /; @RobotsSearchIDOrder = @SearchEnginesSearchIDOrder = (); @_from_p = @_from_h = (); @_time_p = @_time_h = @_time_k = @_time_nv_p = @_time_nv_h = @_time_nv_k = (); @DOWIndex = @fieldlib = @keylist = (); use vars qw/ @MiscListOrder %MiscListCalc @OSFamily %BrowsersFamily @SessionsRange %SessionsAverage %LangBrowserToLangAwstats %LangAWStatsToFlagAwstats @HostAliases @AllowAccessFromWebToFollowingAuthenticatedUsers @DefaultFile @SkipDNSLookupFor @SkipHosts @SkipUserAgents @SkipFiles @OnlyHosts @OnlyUserAgents @OnlyFiles @URLWithQueryWithOnly @URLWithQueryWithout @ExtraName @ExtraCondition @ExtraStatTypes @MaxNbOfExtra @MinHitExtra @ExtraFirstColumnTitle @ExtraFirstColumnValues @ExtraFirstColumnFormat @ExtraCodeFilter @ExtraConditionType @ExtraConditionTypeVal @ExtraFirstColumnValuesType @ExtraFirstColumnValuesTypeVal @ExtraAddAverageRow @ExtraAddSumRow @PluginsToLoad /; @MiscListOrder=('AddToFavourites','JavascriptDisabled','JavaEnabled','DirectorSupport','FlashSupport','RealPlayerSupport','QuickTimeSupport','WindowsMediaPlayerSupport','PDFSupport'); %MiscListCalc=('TotalMisc'=>'','AddToFavourites'=>'u','JavascriptDisabled'=>'hm','JavaEnabled'=>'hm','DirectorSupport'=>'hm','FlashSupport'=>'hm','RealPlayerSupport'=>'hm','QuickTimeSupport'=>'hm','WindowsMediaPlayerSupport'=>'hm','PDFSupport'=>'hm'); @OSFamily=('win','mac'); %BrowsersFamily=('msie'=>1,'firefox'=>2,'netscape'=>3); @SessionsRange=('0s-30s','30s-2mn','2mn-5mn','5mn-15mn','15mn-30mn','30mn-1h','1h+'); %SessionsAverage=('0s-30s',15,'30s-2mn',75,'2mn-5mn',210,'5mn-15mn',600,'15mn-30mn',1350,'30mn-1h',2700,'1h+',3600); # HTTP-Accept or Lang parameter => AWStats code to use for lang # ISO-639-1 or 2 or other => awstats-xx.txt where xx is ISO-639-1 %LangBrowserToLangAwstats=( 'sq'=>'al','ar'=>'ar','ba'=>'ba','bg'=>'bg','zh-tw'=>'tw','zh'=>'cn','cz'=>'cz', 'de'=>'de','da'=>'dk', 'en'=>'en','et'=>'et','fi'=>'fi','fr'=>'fr','gl'=>'gl', 'es'=>'es','eu'=>'eu','ca'=>'ca', 'el'=>'gr','hu'=>'hu','is'=>'is','in'=>'id','it'=>'it', 'ja'=>'jp','ko'=>'kr','lv'=>'lv','nl'=>'nl','no'=>'nb','nb'=>'nb','nn'=>'nn', 'pl'=>'pl','pt'=>'pt','pt-br'=>'br','ro'=>'ro','ru'=>'ru','sr'=>'sr','sk'=>'sk', 'sv'=>'se','th'=>'th','tr'=>'tr','uk'=>'ua','cy'=>'cy','wlk'=>'cy' ); %LangAWStatsToFlagAwstats=( # If flag (country ISO-3166 two letters) is not same than AWStats Lang code 'ca'=>'es_cat','et'=>'ee','eu'=>'es_eu', 'cy'=>'wlk', 'gl'=>'glg', 'he'=>'il', 'ar'=>'sa', 'sr'=>'cs' ); @HostAliases = @AllowAccessFromWebToFollowingAuthenticatedUsers=(); @DefaultFile = @SkipDNSLookupFor = (); @SkipHosts = @SkipUserAgents = @SkipFiles = (); @OnlyHosts = @OnlyUserAgents = @OnlyFiles = (); @URLWithQueryWithOnly = @URLWithQueryWithout = (); @ExtraName = @ExtraCondition = @ExtraStatTypes = @MaxNbOfExtra = @MinHitExtra = (); @ExtraFirstColumnTitle = @ExtraFirstColumnValues = @ExtraFirstColumnFormat = (); @ExtraCodeFilter = @ExtraConditionType = @ExtraConditionTypeVal = (); @ExtraFirstColumnValuesType = @ExtraFirstColumnValuesTypeVal = (); @ExtraAddAverageRow = @ExtraAddSumRow = (); @PluginsToLoad = (); # ---------- Init hash arrays -------- use vars qw/ %BrowsersHashIDLib %BrowsersHashIcon %BrowsersHereAreGrabbers %DomainsHashIDLib %MimeHashLib %MimeHashIcon %MimeHashFamily %OSHashID %OSHashLib %RobotsHashIDLib %RobotsAffiliateLib %SearchEnginesHashID %SearchEnginesHashLib %SearchEnginesWithKeysNotInQuery %SearchEnginesKnownUrl %NotSearchEnginesKeys %WormsHashID %WormsHashLib %WormsHashTarget /; use vars qw/ %HTMLOutput %NoLoadPlugin %FilterIn %FilterEx %BadFormatWarning %MonthNumLib %ValidHTTPCodes %ValidSMTPCodes %TrapInfosForHTTPErrorCodes %NotPageList %DayBytes %DayHits %DayPages %DayVisits %MaxNbOf %MinHit %ListOfYears %HistoryAlreadyFlushed %PosInFile %ValueInFile %val %nextval %egal %TmpDNSLookup %TmpOS %TmpRefererServer %TmpRobot %TmpBrowser %MyDNSTable /; %HTMLOutput = %NoLoadPlugin = %FilterIn = %FilterEx = (); %BadFormatWarning = (); %MonthNumLib = (); %ValidHTTPCodes = %ValidSMTPCodes = (); %TrapInfosForHTTPErrorCodes=(); $TrapInfosForHTTPErrorCodes{404}=1; # TODO Add this in config file %NotPageList=(); %DayBytes = %DayHits = %DayPages = %DayVisits = (); %MaxNbOf = %MinHit = (); %ListOfYears = %HistoryAlreadyFlushed = %PosInFile = %ValueInFile = (); %val = %nextval = %egal = (); %TmpDNSLookup = %TmpOS = %TmpRefererServer = %TmpRobot = %TmpBrowser = %MyDNSTable = (); use vars qw/ %FirstTime %LastTime %MonthHostsKnown %MonthHostsUnknown %MonthUnique %MonthVisits %MonthPages %MonthHits %MonthBytes %MonthNotViewedPages %MonthNotViewedHits %MonthNotViewedBytes %_session %_browser_h %_domener_p %_domener_h %_domener_k %_errors_h %_errors_k %_filetypes_h %_filetypes_k %_filetypes_gz_in %_filetypes_gz_out %_host_p %_host_h %_host_k %_host_l %_host_s %_host_u %_waithost_e %_waithost_l %_waithost_s %_waithost_u %_keyphrases %_keywords %_os_h %_pagesrefs_p %_pagesrefs_h %_robot_h %_robot_k %_robot_l %_robot_r %_worm_h %_worm_k %_worm_l %_login_h %_login_p %_login_k %_login_l %_screensize_h %_misc_p %_misc_h %_misc_k %_cluster_p %_cluster_h %_cluster_k %_se_referrals_p %_se_referrals_h %_sider404_h %_referer404_h %_url_p %_url_k %_url_e %_url_x %_unknownreferer_l %_unknownrefererbrowser_l %_emails_h %_emails_k %_emails_l %_emailr_h %_emailr_k %_emailr_l /; &Init_HashArray(); # ---------- Init Regex -------- use vars qw/ $regclean1 $regclean2 $regdate /; $regclean1=qr/<(recnb|\/td)>/i; $regclean2=qr/<\/?[^<>]+>/i; $regdate=qr/(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)/; # ---------- Init Tie::hash arrays -------- # Didn't find a tie that increase speed #use Tie::StdHash; #use Tie::Cache::LRU; #tie %_host_p, 'Tie::StdHash'; #tie %TmpOS, 'Tie::Cache::LRU'; # PROTOCOL CODES use vars qw/ %httpcodelib %ftpcodelib %smtpcodelib /; # DEFAULT MESSAGE use vars qw/ @Message /; @Message=( 'Unknown', 'Unknown (unresolved ip)', 'Others', 'View details', 'Day', 'Month', 'Year', 'Statistics for', 'First visit', 'Last visit', 'Number of visits', 'Unique visitors', 'Visit', 'different keywords', 'Search', 'Percent', 'Traffic', 'Domains/Countries', 'Visitors', 'Pages-URL', 'Hours', 'Browsers', '', 'Referers', 'Never updated (See \'Build/Update\' on awstats_setup.html page)', 'Visitors domains/countries', 'hosts', 'pages', 'different pages-url', 'Viewed', 'Other words', 'Pages not found', 'HTTP Error codes', 'Netscape versions', 'IE versions', 'Last Update', 'Connect to site from', 'Origin', 'Direct address / Bookmarks', 'Origin unknown', 'Links from an Internet Search Engine', 'Links from an external page (other web sites except search engines)', 'Links from an internal page (other page on same site)', 'Keyphrases used on search engines', 'Keywords used on search engines', 'Unresolved IP Address', 'Unknown OS (Referer field)', 'Required but not found URLs (HTTP code 404)', 'IP Address', 'Error Hits', 'Unknown browsers (Referer field)', 'different robots', 'visits/visitor', 'Robots/Spiders visitors', 'Free realtime logfile analyzer for advanced web statistics', 'of', 'Pages', 'Hits', 'Versions', 'Operating Systems', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec', 'Navigation', 'File type', 'Update now', 'Bandwidth', 'Back to main page', 'Top', 'dd mmm yyyy - HH:MM', 'Filter', 'Full list', 'Hosts', 'Known', 'Robots', 'Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Days of week', 'Who', 'When', 'Authenticated users', 'Min', 'Average', 'Max', 'Web compression', 'Bandwidth saved', 'Compression on', 'Compression result', 'Total', 'different keyphrases', 'Entry', 'Code', 'Average size', 'Links from a NewsGroup', 'KB', 'MB', 'GB', 'Grabber', 'Yes', 'No', 'Info.', 'OK', 'Exit', 'Visits duration', 'Close window', 'Bytes', 'Search Keyphrases', 'Search Keywords', 'different refering search engines', 'different refering sites', 'Other phrases', 'Other logins (and/or anonymous users)', 'Refering search engines', 'Refering sites', 'Summary', 'Exact value not available in "Year" view', 'Data value arrays', 'Sender EMail', 'Receiver EMail', 'Reported period', 'Extra/Marketing', 'Screen sizes', 'Worm/Virus attacks', 'Add to favorites (estimated)', 'Days of month', 'Miscellaneous', 'Browsers with Java support', 'Browsers with Macromedia Director Support', 'Browsers with Flash Support', 'Browsers with Real audio playing support', 'Browsers with Quictime audio playing support', 'Browsers with Windows Media audio playing support', 'Browsers with PDF support', 'SMTP Error codes', 'Countries', 'Mails', 'Size', 'First', 'Last', 'Exclude filter', 'Codes shown here gave hits or traffic "not viewed" by visitors, so they are not included in other charts.', 'Cluster', 'Robots shown here gave hits or traffic "not viewed" by visitors, so they are not included in other charts.', 'Numbers after + are successful hits on "robots.txt" files', 'Worms shown here gave hits or traffic "not viewed" by visitors, so thay are not included in other charts.', 'Not viewed traffic includes traffic generated by robots, worms, or replies with special HTTP status codes.', 'Traffic viewed', 'Traffic not viewed', 'Monthly history', 'Worms', 'different worms', 'Mails successfully sent', 'Mails failed/refused', 'Sensitive targets', 'Javascript disabled', 'Created by', 'plugins' ); #------------------------------------------------------------------------------ # Functions #------------------------------------------------------------------------------ #------------------------------------------------------------------------------ # Function: Write on ouput header of HTTP answer # Parameters: None # Input: $HeaderHTTPSent $BuildReportFormat $PageCode $Expires # Output: $HeaderHTTPSent=1 # Return: None #------------------------------------------------------------------------------ sub http_head { if (! $HeaderHTTPSent) { if ($BuildReportFormat eq 'xhtml' || $BuildReportFormat eq 'xml') { print ($ENV{'HTTP_USER_AGENT'}=~/MSIE|Googlebot/i?"Content-type: text/html; charset=$PageCode\n":"Content-type: text/xml; charset=$PageCode\n"); } else { print "Content-type: text/html; charset=$PageCode\n"; } # Expires must be GMT ANSI asctime and must be after Content-type to avoid pb with some servers (SAMBAR) if ($Expires =~ /^\d+$/) { print "Cache-Control: public\n"; print "Last-Modified: ".gmtime($starttime)."\n"; print "Expires: ".(gmtime($starttime+$Expires))."\n"; } print "\n"; } $HeaderHTTPSent++;; } #------------------------------------------------------------------------------ # Function: Write on ouput header of HTML page # Parameters: None # Input: %HTMLOutput $PluginMode $Expires $Lang $StyleSheet $HTMLHeadSection $PageCode $PageDir # Output: $HeaderHTMLSent=1 # Return: None #------------------------------------------------------------------------------ sub html_head { my $dir=$PageDir?'right':'left'; if (scalar keys %HTMLOutput || $PluginMode) { my $MetaRobot=0; # meta robots my $periodtitle=" ($YearRequired".($MonthRequired ne 'all'?"-$MonthRequired":"").")"; # Write head section if ($BuildReportFormat eq 'xhtml' || $BuildReportFormat eq 'xml') { if ($PageCode) { print "\n"; } else { print "\n"; }; if ($FrameName ne 'index') { print "\n"; } else { print "\n"; } print "\n"; } else { if ($FrameName ne 'index') { print "\n\n"; } else { print "\n\n"; } print "\n"; } print "\n"; print "\n"; if ($MetaRobot) { print "\n"; } else { print "\n"; } # Affiche tag meta content-type if ($BuildReportFormat eq 'xhtml' || $BuildReportFormat eq 'xml') { print ($ENV{'HTTP_USER_AGENT'}=~/MSIE|Googlebot/i?"\n":"\n"); } else { print "\n"; } if ($Expires) { print "\n"; } print "\n"; if ($MetaRobot && $FrameName ne 'mainleft') { print "\n"; } print "$Message[7] $SiteDomain$periodtitle\n"; if ($FrameName ne 'index') { # A STYLE section must be in head section. Do not use " for number in a style section print "\n"; if ($StyleSheet) { print "\n"; } } print "\n\n"; if ($FrameName ne 'index') { print "\n"; } } $HeaderHTMLSent++; } #------------------------------------------------------------------------------ # Function: Write on ouput end of HTML page # Parameters: 0|1 (0=no list plugins,1=list plugins) # Input: %HTMLOutput $HTMLEndSection $FrameName $BuildReportFormat # Output: None # Return: None #------------------------------------------------------------------------------ sub html_end { my $listplugins=shift||0; if (scalar keys %HTMLOutput) { # Call to plugins' function AddHTMLBodyFooter foreach my $pluginname (keys %{$PluginsLoaded{'AddHTMLBodyFooter'}}) { my $function="AddHTMLBodyFooter_$pluginname()"; eval("$function"); } if ($FrameName ne 'index' && $FrameName ne 'mainleft') { print "$Center

\n"; print ""; print "Advanced Web Statistics $VERSION - "; print $Message[169]." $PROG"; if ($listplugins) { my $atleastoneplugin=0; foreach my $pluginname (keys %{$PluginsLoaded{'init'}}) { if (! $atleastoneplugin) { $atleastoneplugin=1; print " ($Message[170]: "; } else { print ", "; } print "$pluginname"; } if ($atleastoneplugin) { print ")"; } } print "
\n"; if ($HTMLEndSection) { print "
\n$HTMLEndSection\n"; } } print "\n"; if ($FrameName ne 'index') { if ($FrameName ne 'mainleft' && $BuildReportFormat eq 'html') { print "
\n"; } print "\n"; } print "\n"; # print "\n"; } } #------------------------------------------------------------------------------ # Function: Print on stdout tab header of a chart # Parameters: $title $tooltipnb [$width percentage of chart title] # Input: None # Output: None # Return: None #------------------------------------------------------------------------------ sub tab_head { my $title=shift; my $tooltipnb=shift; my $width=shift||70; my $class=shift; if ($width == 70 && $QueryString =~ /buildpdf/i) { print "\n"; } else { print "
\n"; } if ($tooltipnb) { print ""; } else { print ""; } print "\n"; print "
$title
$title  
\n"; if ($width == 70 && $QueryString =~ /buildpdf/i) { print "\n"; } else { print "
\n"; } } #------------------------------------------------------------------------------ # Function: Print on stdout tab ender of a chart # Parameters: None # Input: None # Output: None # Return: None #------------------------------------------------------------------------------ sub tab_end { my $string=shift; print "
"; if ($string) { print "$string
\n"; } print "
\n\n"; } #------------------------------------------------------------------------------ # Function: Write error message and exit # Parameters: $message $secondmessage $thirdmessage $donotshowsetupinfo # Input: $HeaderHTTPSent $HeaderHTMLSent %HTMLOutput $LogSeparator $LogFormat # Output: None # Return: None #------------------------------------------------------------------------------ sub error { my $message=shift||''; if (scalar keys %HTMLOutput) { $message =~ s/\/>/g; } my $secondmessage=shift||''; my $thirdmessage=shift||''; my $donotshowsetupinfo=shift||0; if (! $HeaderHTTPSent && $ENV{'GATEWAY_INTERFACE'}) { http_head(); } if (! $HeaderHTMLSent && scalar keys %HTMLOutput) { print "\n"; $HeaderHTMLSent=1; } if ($Debug) { debug("$message $secondmessage $thirdmessage",1); } my $tagbold=''; my $tagunbold=''; my $tagbr=''; my $tagfontred=''; my $tagfontgrey=''; my $tagunfont=''; if (scalar keys %HTMLOutput) { $tagbold=''; $tagunbold=''; $tagbr='
'; $tagfontred=''; $tagfontgrey=''; $tagunfont=''; } if (! $ErrorMessages && $message =~ /^Format error$/i) { # Files seems to have bad format if (scalar keys %HTMLOutput) { print "

\n"; } if ($message !~ $LogSeparator) { # Bad LogSeparator parameter print "${tagfontred}AWStats did not found the ${tagbold}LogSeparator${tagunbold} in your log records.${tagbr}${tagunfont}\n"; } else { # Bad LogFormat parameter print "AWStats did not find any valid log lines that match your ${tagbold}LogFormat${tagunbold} parameter, in the ${NbOfLinesForCorruptedLog}th first non commented lines read of your log.${tagbr}\n"; print "${tagfontred}Your log file ${tagbold}$thirdmessage${tagunbold} must have a bad format or ${tagbold}LogFormat${tagunbold} parameter setup does not match this format.${tagbr}${tagbr}${tagunfont}\n"; print "Your AWStats ${tagbold}LogFormat${tagunbold} parameter is:\n"; print "${tagbold}$LogFormat${tagunbold}${tagbr}\n"; print "This means each line in your web server log file need to have "; if ($LogFormat == 1) { print "${tagbold}\"combined log format\"${tagunbold} like this:${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "111.22.33.44 - - [10/Jan/2001:02:14:14 +0200] \"GET / HTTP/1.1\" 200 1234 \"http://www.fromserver.com/from.htm\" \"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)\"\n"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}\n":""); } if ($LogFormat == 2) { print "${tagbold}\"MSIE Extended W3C log format\"${tagunbold} like this:${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "date time c-ip c-username cs-method cs-uri-sterm sc-status sc-bytes cs-version cs(User-Agent) cs(Referer)\n"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}\n":""); } if ($LogFormat == 3) { print "${tagbold}\"WebStar native log format\"${tagunbold}${tagbr}\n"; } if ($LogFormat == 4) { print "${tagbold}\"common log format\"${tagunbold} like this:${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "111.22.33.44 - - [10/Jan/2001:02:14:14 +0200] \"GET / HTTP/1.1\" 200 1234\n"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}\n":""); } if ($LogFormat == 5) { print "${tagbold}\"ISA native log format\"${tagunbold}${tagbr}\n"; } if ($LogFormat == 6) { print "${tagbold}\"Lotus Notes/Lotus Domino\"${tagunbold}${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "111.22.33.44 - Firstname Middlename Lastname [10/Jan/2001:02:14:14 +0200] \"GET / HTTP/1.1\" 200 1234 \"http://www.fromserver.com/from.htm\" \"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)\"\n"; print (scalar keys %HTMLOutput?"${tagbr}${tagbr}\n":""); } if ($LogFormat !~ /^[1-6]$/) { print "the following personalized log format:${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "$LogFormat\n"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}\n":""); } print "And this is an example of records AWStats found in your log file (the record number $NbOfLinesForCorruptedLog in your log):\n"; print (scalar keys %HTMLOutput?"
$tagfontgrey":""); print "$secondmessage"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}":""); print "\n"; } #print "Note: If your $NbOfLinesForCorruptedLog first lines in your log files are wrong because of "; #print "a worm virus attack, you can increase the NbOfLinesForCorruptedLog parameter in config file.\n"; #print "\n"; } else { print (scalar keys %HTMLOutput?"
$tagfontred\n":""); print ($ErrorMessages?"$ErrorMessages":"Error: $message"); print (scalar keys %HTMLOutput?"\n
":""); print "\n"; } if (! $ErrorMessages && ! $donotshowsetupinfo) { if ($message =~ /Couldn.t open config file/i) { my $dir=$DIR; if ($dir =~ /^\./) { $dir.='/../..'; } else { $dir =~ s/[\\\/]?wwwroot[\/\\]cgi-bin[\\\/]?//; } print "${tagbr}\n"; if ($ENV{'GATEWAY_INTERFACE'}) { print "- ${tagbold}Did you use the correct URL ?${tagunbold}${tagbr}\n"; print "Example: http://localhost/awstats/awstats.pl?config=mysite${tagbr}\n"; print "Example: http://127.0.0.1/cgi-bin/awstats.pl?config=mysite${tagbr}\n"; } else { print "- ${tagbold}Did you use correct config parameter ?${tagunbold}${tagbr}\n"; print "Example: If your config file is awstats.mysite.conf, use -config=mysite\n"; } print "- ${tagbold}Did you create your config file 'awstats.$SiteConfig.conf' ?${tagunbold}${tagbr}\n"; print "If not, you can run \"$dir/tools/awstats_configure.pl\"\nfrom command line, or create it manually.${tagbr}\n"; print "${tagbr}\n"; } else { print "${tagbr}${tagbold}Setup (".($FileConfig?"'".$FileConfig."'":"Config")." file, web server or permissions) may be wrong.${tagunbold}${tagbr}\n"; } print "Check config file, permissions and AWStats documentation (in 'docs' directory).\n"; } # Remove lock if not a lock message if ($EnableLockForUpdate && $message !~ /lock file/) { &Lock_Update(0); } if (scalar keys %HTMLOutput) { print "\n"; } exit 1; } #------------------------------------------------------------------------------ # Function: Write a warning message # Parameters: $message # Input: $HeaderHTTPSent $HeaderHTMLSent $WarningMessage %HTMLOutput # Output: None # Return: None #------------------------------------------------------------------------------ sub warning { my $messagestring=shift; if (! $HeaderHTTPSent && $ENV{'GATEWAY_INTERFACE'}) { http_head(); } if (! $HeaderHTMLSent) { html_head(); } if ($Debug) { debug("$messagestring",1); } if ($WarningMessages) { if (scalar keys %HTMLOutput) { $messagestring =~ s/\n/\/g; print "$messagestring
\n"; } else { print "$messagestring\n"; } } } #------------------------------------------------------------------------------ # Function: Write debug message and exit # Parameters: $string $level # Input: %HTMLOutput $Debug=required level $DEBUGFORCED=required level forced # Output: None # Return: None #------------------------------------------------------------------------------ sub debug { my $level = $_[1] || 1; if (! $HeaderHTTPSent && $ENV{'GATEWAY_INTERFACE'}) { http_head(); } # To send the HTTP header and see debug if ($level <= $DEBUGFORCED) { my $debugstring = $_[0]; if (! $DebugResetDone) { open(DEBUGFORCEDFILE,"debug.log"); close DEBUGFORCEDFILE; chmod 0666,"debug.log"; $DebugResetDone=1; } open(DEBUGFORCEDFILE,">>debug.log"); print DEBUGFORCEDFILE localtime(time)." - $$ - DEBUG $level - $debugstring\n"; close DEBUGFORCEDFILE; } if ($DebugMessages && $level <= $Debug) { my $debugstring = $_[0]; if (scalar keys %HTMLOutput) { $debugstring =~ s/^ /   /; $debugstring .= "
"; } print localtime(time)." - DEBUG $level - $debugstring\n"; } } #------------------------------------------------------------------------------ # Function: Optimize an array removing duplicate entries # Parameters: @Array notcasesensitive=0|1 # Input: None # Output: None # Return: None #------------------------------------------------------------------------------ sub OptimizeArray { my $array=shift; my @arrayunreg=map{if (/\(\?[-\w]*:(.*)\)/) { $1 } } @$array; my $notcasesensitive=shift; my $searchlist=0; if ($Debug) { debug("OptimizeArray (notcasesensitive=$notcasesensitive)",4); } while ($searchlist>-1 && @arrayunreg) { my $elemtoremove=-1; OPTIMIZELOOP: foreach my $i ($searchlist..(scalar @arrayunreg)-1) { # Search if $i elem is already treated by another elem foreach my $j (0..(scalar @arrayunreg)-1) { if ($i == $j) { next; } my $parami=$notcasesensitive?lc($arrayunreg[$i]):$arrayunreg[$i]; my $paramj=$notcasesensitive?lc($arrayunreg[$j]):$arrayunreg[$j]; if ($Debug) { debug(" Compare $i ($parami) to $j ($paramj)",4); } if (index($parami,$paramj)>-1) { if ($Debug) { debug(" Elem $i ($arrayunreg[$i]) already treated with elem $j ($arrayunreg[$j])",4); } $elemtoremove=$i; last OPTIMIZELOOP; } } } if ($elemtoremove > -1) { if ($Debug) { debug(" Remove elem $elemtoremove - $arrayunreg[$elemtoremove]",4); } splice @arrayunreg, $elemtoremove, 1; $searchlist=$elemtoremove; } else { $searchlist=-1; } } if ($notcasesensitive) { return map{qr/$_/i} @arrayunreg; } return map{qr/$_/} @arrayunreg; } #------------------------------------------------------------------------------ # Function: Check if parameter is in SkipDNSLookupFor array # Parameters: ip @SkipDNSLookupFor (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub SkipDNSLookup { foreach (@SkipDNSLookupFor) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @SkipDNSLookupFor } #------------------------------------------------------------------------------ # Function: Check if parameter is in SkipHosts array # Parameters: host @SkipHosts (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub SkipHost { foreach (@SkipHosts) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @SkipHosts } #------------------------------------------------------------------------------ # Function: Check if parameter is in SkipUserAgents array # Parameters: useragent @SkipUserAgents (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub SkipUserAgent { foreach (@SkipUserAgents) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @SkipUserAgent } #------------------------------------------------------------------------------ # Function: Check if parameter is in SkipFiles array # Parameters: url @SkipFiles (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub SkipFile { foreach (@SkipFiles) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @SkipFiles } #------------------------------------------------------------------------------ # Function: Check if parameter is in OnlyHosts array # Parameters: host @OnlyHosts (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub OnlyHost { foreach (@OnlyHosts) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @OnlyHosts } #------------------------------------------------------------------------------ # Function: Check if parameter is in OnlyUserAgents array # Parameters: useragent @OnlyUserAgents (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub OnlyUserAgent { foreach (@OnlyUserAgents) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @OnlyHosts } #------------------------------------------------------------------------------ # Function: Check if parameter is in OnlyFiles array # Parameters: url @OnlyFiles (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub OnlyFile { foreach (@OnlyFiles) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @OnlyFiles } #------------------------------------------------------------------------------ # Function: Return day of week of a day # Parameters: $day $month $year # Return: 0-6 #------------------------------------------------------------------------------ sub DayOfWeek { my ($day, $month, $year) = @_; if ($Debug) { debug("DayOfWeek for $day $month $year",4); } if ($month < 3) { $month += 10; $year--; } else { $month -= 2; } my $cent = sprintf("%1i",($year/100)); my $y = ($year % 100); my $dw = (sprintf("%1i",(2.6*$month)-0.2) + $day + $y + sprintf("%1i",($y/4)) + sprintf("%1i",($cent/4)) - (2*$cent)) % 7; $dw += 7 if ($dw<0); if ($Debug) { debug(" is $dw",4); } return $dw; } #------------------------------------------------------------------------------ # Function: Return 1 if a date exists # Parameters: $day $month $year # Return: 1 if date exists else 0 #------------------------------------------------------------------------------ sub DateIsValid { my ($day, $month, $year) = @_; if ($Debug) { debug("DateIsValid for $day $month $year",4); } if ($day < 1) { return 0; } if ($day > 31) { return 0; } if ($month==4 || $month==6 || $month==9 || $month==11) { if ($day > 30) { return 0; } } elsif ($month==2) { my $leapyear=($year%4==0?1:0); # A leap year every 4 years if ($year%100==0 && $year%400!=0) { $leapyear=0; } # Except if year is 100x and not 400x if ($day > (28+$leapyear)) { return 0; } } return 1; } #------------------------------------------------------------------------------ # Function: Return string of visit duration # Parameters: $starttime $endtime # Input: None # Output: None # Return: A string that identify the visit duration range #------------------------------------------------------------------------------ sub GetSessionRange { my $starttime = my $endtime; if (shift =~ /$regdate/o) { $starttime = Time::Local::timelocal($6,$5,$4,$3,$2-1,$1); } if (shift =~ /$regdate/o) { $endtime = Time::Local::timelocal($6,$5,$4,$3,$2-1,$1); } my $delay=$endtime-$starttime; if ($Debug) { debug("GetSessionRange $endtime - $starttime = $delay",4); } if ($delay <= 30) { return $SessionsRange[0]; } if ($delay <= 120) { return $SessionsRange[1]; } if ($delay <= 300) { return $SessionsRange[2]; } if ($delay <= 900) { return $SessionsRange[3]; } if ($delay <= 1800) { return $SessionsRange[4]; } if ($delay <= 3600) { return $SessionsRange[5]; } return $SessionsRange[6]; } #------------------------------------------------------------------------------ # Function: Read config file # Parameters: None or configdir to scan # Input: $DIR $PROG $SiteConfig # Output: Global variables # Return: - #------------------------------------------------------------------------------ sub Read_Config { # Check config file in common possible directories : # Windows : "$DIR" (same dir than awstats.pl) # Standard, Mandrake and Debian package : "/etc/awstats" # Other possible directories : "/usr/local/etc/awstats", "/etc" # FHS standard, Suse package : "/etc/opt/awstats" my $configdir=shift; my @PossibleConfigDir=(); if ($configdir) { @PossibleConfigDir=("$configdir"); } else { @PossibleConfigDir=("$DIR","/etc/awstats","/usr/local/etc/awstats","/etc","/etc/opt/awstats"); } # Open config file $FileConfig=$FileSuffix=''; foreach (@PossibleConfigDir) { my $searchdir=$_; if ($searchdir && $searchdir !~ /[\\\/]$/) { $searchdir .= "/"; } if (open(CONFIG,"$searchdir$PROG.$SiteConfig.conf")) { $FileConfig="$searchdir$PROG.$SiteConfig.conf"; $FileSuffix=".$SiteConfig"; last; } if (open(CONFIG,"$searchdir$PROG.conf")) { $FileConfig="$searchdir$PROG.conf"; $FileSuffix=''; last; } } if (! $FileConfig) { error("Couldn't open config file \"$PROG.$SiteConfig.conf\" nor \"$PROG.conf\" after searching in path \"".join(',',@PossibleConfigDir)."\": $!"); } # Analyze config file content and close it &Parse_Config( *CONFIG , 1 , $FileConfig); close CONFIG; # If parameter NotPageList not found, init for backward compatibility if (! $FoundNotPageList) { %NotPageList=('css'=>1,'js'=>1,'class'=>1,'gif'=>1,'jpg'=>1,'jpeg'=>1,'png'=>1,'bmp'=>1,'ico'=>1,'swf'=>1); } # If parameter ValidHTTPCodes empty, init for backward compatibility if (! scalar keys %ValidHTTPCodes) { $ValidHTTPCodes{"200"}=$ValidHTTPCodes{"304"}=1; } # If parameter ValidSMTPCodes empty, init for backward compatibility if (! scalar keys %ValidSMTPCodes) { $ValidSMTPCodes{"1"}=$ValidSMTPCodes{"250"}=1; } } #------------------------------------------------------------------------------ # Function: Parse content of a config file # Parameters: opened file handle, depth level, file name # Input: - # Output: Global variables # Return: - #------------------------------------------------------------------------------ sub Parse_Config { my ( $confighandle ) = $_[0]; my $level = $_[1]; my $configFile = $_[2]; my $versionnum=0; my $conflinenb=0; if ($level > 10) { error("$PROG can't read down more than 10 level of includes. Check that no 'included' config files include their parent config file (this cause infinite loop)."); } while (<$confighandle>) { chomp $_; s/\r//; $conflinenb++; # Extract version from first line if (! $versionnum && $_ =~ /^# AWSTATS CONFIGURE FILE (\d+).(\d+)/i) { $versionnum=($1*1000)+$2; #if ($Debug) { debug(" Configure file version is $versionnum",1); } next; } if ($_ =~ /^\s*$/) { next; } # Check includes if ($_ =~ /^Include "([^\"]+)"/ || $_ =~ /^#include "([^\"]+)"/) { # #include kept for backward compatibility my $includeFile = $1; if ($Debug) { debug("Found an include : $includeFile",2); } if ( $includeFile !~ /^[\\\/]/ ) { # Correct relative include files if ($FileConfig =~ /^(.*[\\\/])[^\\\/]*$/) { $includeFile = "$1$includeFile"; } } if ($level > 1) { warning("Warning: Perl versions before 5.6 cannot handle nested includes"); next; } if ( open( CONFIG_INCLUDE, $includeFile ) ) { &Parse_Config( *CONFIG_INCLUDE , $level+1, $includeFile); close( CONFIG_INCLUDE ); } else { error("Could not open include file: $includeFile" ); } next; } # Remove comments if ($_ =~ /^\s*#/) { next; } $_ =~ s/\s#.*$//; # Extract param and value my ($param,$value)=split(/=/,$_,2); $param =~ s/^\s+//; $param =~ s/\s+$//; # If not a param=value, try with next line if (! $param) { warning("Warning: Syntax error line $conflinenb in file '$configFile'. Config line is ignored."); next; } if (! defined $value) { warning("Warning: Syntax error line $conflinenb in file '$configFile'. Config line is ignored."); next; } if ($value) { $value =~ s/^\s+//; $value =~ s/\s+$//; $value =~ s/^\"//; $value =~ s/\";?$//; # Replace __MONENV__ with value of environnement variable MONENV # Must be able to replace __VAR_1____VAR_2__ while ($value =~ /__([^\s_]+(?:_[^\s_]+)*)__/) { my $var=$1; $value =~ s/__${var}__/$ENV{$var}/g; } } # Initialize parameter for (param,value) if ($param =~ /^LogFile/) { if ($QueryString !~ /logfile=([^\s&]+)/i) { $LogFile=$value; } next; } if ($param =~ /^DirIcons/) { if ($QueryString !~ /diricons=([^\s&]+)/i) { $DirIcons=$value; } next; } if ($param =~ /^SiteDomain/) { # No regex test as SiteDomain is always exact value $SiteDomain=$value; next; } if ($param =~ /^HostAliases/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ s/^\@//) { # If list of hostaliases in a file open(DATAFILE,"<$elem") || error("Failed to open file '$elem' declared in HostAliases parameter"); my @val=map(/^(.*)$/i,); push @HostAliases, map{qr/^$_$/i} @val; close(DATAFILE); } else { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @HostAliases, qr/$elem/i; } } } next; } # Special optional setup params if ($param =~ /^SkipDNSLookupFor/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @SkipDNSLookupFor, qr/$elem/i; } } next; } if ($param =~ /^AllowAccessFromWebToFollowingAuthenticatedUsers/) { foreach (split(/\s+/,$value)) { push @AllowAccessFromWebToFollowingAuthenticatedUsers,$_; } next; } if ($param =~ /^DefaultFile/) { foreach my $elem (split(/\s+/,$value)) { # No REGEX for this option #if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } #else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @DefaultFile,$elem; } } next; } if ($param =~ /^SkipHosts/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @SkipHosts, qr/$elem/i; } } next; } if ($param =~ /^SkipUserAgents/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @SkipUserAgents, qr/$elem/i; } } next; } if ($param =~ /^SkipFiles/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @SkipFiles, qr/$elem/i; } } next; } if ($param =~ /^OnlyHosts/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyHosts, qr/$elem/i; } } next; } if ($param =~ /^OnlyUserAgents/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyUserAgents, qr/$elem/i; } } next; } if ($param =~ /^OnlyFiles/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyFiles, qr/$elem/i; } } next; } if ($param =~ /^NotPageList/) { foreach (split(/\s+/,$value)) { $NotPageList{$_}=1; } $FoundNotPageList=1; next; } if ($param =~ /^ValidHTTPCodes/) { foreach (split(/\s+/,$value)) { $ValidHTTPCodes{$_}=1; } next; } if ($param =~ /^ValidSMTPCodes/) { foreach (split(/\s+/,$value)) { $ValidSMTPCodes{$_}=1; } next; } if ($param =~ /^URLWithQueryWithOnlyFollowingParameters$/) { @URLWithQueryWithOnly=split(/\s+/,$value); next; } if ($param =~ /^URLWithQueryWithoutFollowingParameters$/) { @URLWithQueryWithout=split(/\s+/,$value); next; } # Extra parameters if ($param =~ /^ExtraSectionName(\d+)/) { $ExtraName[$1]=$value; next; } if ($param =~ /^ExtraSectionCodeFilter(\d+)/) { @{$ExtraCodeFilter[$1]}=split(/\s+/,$value); next; } if ($param =~ /^ExtraSectionCondition(\d+)/) { $ExtraCondition[$1]=$value; next; } if ($param =~ /^ExtraSectionStatTypes(\d+)/) { $ExtraStatTypes[$1]=$value; next; } if ($param =~ /^ExtraSectionFirstColumnTitle(\d+)/) { $ExtraFirstColumnTitle[$1]=$value; next; } if ($param =~ /^ExtraSectionFirstColumnValues(\d+)/) { $ExtraFirstColumnValues[$1]=$value; next; } if ($param =~ /^ExtraSectionFirstColumnFormat(\d+)/) { $ExtraFirstColumnFormat[$1]=$value; next; } if ($param =~ /^ExtraSectionAddAverageRow(\d+)/) { $ExtraAddAverageRow[$1]=$value; next; } if ($param =~ /^ExtraSectionAddSumRow(\d+)/) { $ExtraAddSumRow[$1]=$value; next; } if ($param =~ /^MaxNbOfExtra(\d+)/) { $MaxNbOfExtra[$1]=$value; next; } if ($param =~ /^MinHitExtra(\d+)/) { $MinHitExtra[$1]=$value; next; } # Special appearance parameters if ($param =~ /^LoadPlugin/) { push @PluginsToLoad, $value; next; } # Other parameter checks we need to put after MaxNbOfExtra and MinHitExtra if ($param =~ /^MaxNbOf(\w+)/) { $MaxNbOf{$1}=$value; next; } if ($param =~ /^MinHit(\w+)/) { $MinHit{$1}=$value; next; } # Check if this is a known parameter # if (! $ConfOk{$param}) { error("Unknown config parameter '$param' found line $conflinenb in file \"configFile\""); } # If parameters was not found previously, defined variable with name of param to value $$param=$value; } # For backward compatibility if ($versionnum < 5001) { $BarHeight=$BarHeight>>1; } if ($Debug) { debug("Config file read was \"$configFile\" (level $level)"); } } #------------------------------------------------------------------------------ # Function: Load the reference databases # Parameters: List of files to load # Input: $DIR # Output: Arrays and Hash tables are defined # Return: - #------------------------------------------------------------------------------ sub Read_Ref_Data { # Check lib files in common possible directories : # Windows and standard package: "$DIR/lib" (lib in same dir than awstats.pl) # Debian package: "/usr/share/awstats/lib" my @PossibleLibDir=("$DIR/lib","/usr/share/awstats/lib"); my %FilePath=(); my %DirAddedInINC=(); my @FileListToLoad=(); while (my $file=shift) { push @FileListToLoad, "$file.pm"; } if ($Debug) { debug("Call to Read_Ref_Data with files to load: ".(join(',',@FileListToLoad))); } foreach my $file (@FileListToLoad) { foreach my $dir (@PossibleLibDir) { my $searchdir=$dir; if ($searchdir && (!($searchdir =~ /\/$/)) && (!($searchdir =~ /\\$/)) ) { $searchdir .= "/"; } if (! $FilePath{$file}) { # To not load twice same file in different path if (-s "${searchdir}${file}") { $FilePath{$file}="${searchdir}${file}"; if ($Debug) { debug("Call to Read_Ref_Data [FilePath{$file}=\"$FilePath{$file}\"]"); } # Note: cygwin perl 5.8 need a push + require file if (! $DirAddedInINC{"$dir"}) { push @INC, "$dir"; $DirAddedInINC{"$dir"}=1; } my $loadret=require "$file"; #my $loadret=(require "$FilePath{$file}"||require "${file}"); } } } if (! $FilePath{$file}) { my $filetext=$file; $filetext =~ s/\.pm$//; $filetext =~ s/_/ /g; warning("Warning: Can't read file \"$file\" ($filetext detection will not work correctly).\nCheck if file is in \"".($PossibleLibDir[0])."\" directory and is readable."); } } # Sanity check (if loaded) if ((scalar keys %OSHashID) && @OSSearchIDOrder != scalar keys %OSHashID) { error("Not same number of records of OSSearchIDOrder (".(@OSSearchIDOrder)." entries) and OSHashID (".(scalar keys %OSHashID)." entries) in OS database. Check your file ".$FilePath{"operating_systems.pm"}); } if ((scalar keys %SearchEnginesHashID) && (@SearchEnginesSearchIDOrder_list1+@SearchEnginesSearchIDOrder_list2+@SearchEnginesSearchIDOrder_listgen) != scalar keys %SearchEnginesHashID) { error("Not same number of records of SearchEnginesSearchIDOrder_listx (total is ".(@SearchEnginesSearchIDOrder_list1+@SearchEnginesSearchIDOrder_list2+@SearchEnginesSearchIDOrder_listgen)." entries) and SearchEnginesHashID (".(scalar keys %SearchEnginesHashID)." entries) in Search Engines database. Check your file ".$FilePath{"search_engines.pm"}." is up to date."); } if ((scalar keys %BrowsersHashIDLib) && @BrowsersSearchIDOrder != (scalar keys %BrowsersHashIDLib) - 3) { error("Not same number of records of BrowsersSearchIDOrder (".(@BrowsersSearchIDOrder)." entries) and BrowsersHashIDLib (".((scalar keys %BrowsersHashIDLib) - 3)." entries without msie,netscape,firefox) in Browsers database. May be you updated AWStats without updating browsers.pm file or you made changed into browsers.pm not correctly. Check your file ".$FilePath{"browsers.pm"}." is up to date."); } if ((scalar keys %RobotsHashIDLib) && (@RobotsSearchIDOrder_list1+@RobotsSearchIDOrder_list2+@RobotsSearchIDOrder_listgen) != (scalar keys %RobotsHashIDLib) - 1) { error("Not same number of records of RobotsSearchIDOrder_listx (total is ".(@RobotsSearchIDOrder_list1+@RobotsSearchIDOrder_list2+@RobotsSearchIDOrder_listgen)." entries) and RobotsHashIDLib (".((scalar keys %RobotsHashIDLib) - 1)." entries without 'unknown') in Robots database. Check your file ".$FilePath{"robots.pm"}." is up to date."); } } #------------------------------------------------------------------------------ # Function: Get the messages for a specified language # Parameters: LanguageId # Input: $DirLang $DIR # Output: $Message table is defined in memory # Return: None #------------------------------------------------------------------------------ sub Read_Language_Data { # Check lang files in common possible directories : # Windows and standard package: "$DIR/lang" (lang in same dir than awstats.pl) # Debian package : "/usr/share/awstats/lang" my @PossibleLangDir=("$DirLang","$DIR/lang","/usr/share/awstats/lang"); my $FileLang=''; foreach (@PossibleLangDir) { my $searchdir=$_; if ($searchdir && (!($searchdir =~ /\/$/)) && (!($searchdir =~ /\\$/)) ) { $searchdir .= "/"; } if (open(LANG,"${searchdir}awstats-$_[0].txt")) { $FileLang="${searchdir}awstats-$_[0].txt"; last; } } # If file not found, we try english if (! $FileLang) { foreach (@PossibleLangDir) { my $searchdir=$_; if ($searchdir && (!($searchdir =~ /\/$/)) && (!($searchdir =~ /\\$/)) ) { $searchdir .= "/"; } if (open(LANG,"${searchdir}awstats-en.txt")) { $FileLang="${searchdir}awstats-en.txt"; last; } } } if ($Debug) { debug("Call to Read_Language_Data [FileLang=\"$FileLang\"]"); } if ($FileLang) { my $i = 0; binmode LANG; # Might avoid 'Malformed UTF-8 errors' my $cregcode=qr/^PageCode=[\t\s\"\']*([\w-]+)/i; my $cregdir=qr/^PageDir=[\t\s\"\']*([\w-]+)/i; my $cregmessage=qr/^Message\d+=/i; while () { chomp $_; s/\r//; if ($_ =~ /$cregcode/o) { $PageCode = $1; } if ($_ =~ /$cregdir/o) { $PageDir = $1; } if ($_ =~ s/$cregmessage//o) { $_ =~ s/^#.*//; # Remove comments $_ =~ s/\s+#.*//; # Remove comments $_ =~ tr/\t / /s; # Change all blanks into " " $_ =~ s/^\s+//; $_ =~ s/\s+$//; $_ =~ s/^\"//; $_ =~ s/\"$//; $Message[$i] = $_; $i++; } } close(LANG); } else { warning("Warning: Can't find language files for \"$_[0]\". English will be used."); } # Some language string changes if ($LogType eq 'M') { # For mail $Message[8]=$Message[151]; $Message[9]=$Message[152]; $Message[57]=$Message[149]; $Message[75]=$Message[150]; } if ($LogType eq 'F') { # For web } } #------------------------------------------------------------------------------ # Function: Substitute date tags in a string by value # Parameters: String # Input: All global variables # Output: Change on some global variables # Return: String #------------------------------------------------------------------------------ sub Substitute_Tags { my $SourceString=shift; if ($Debug) { debug("Call to Substitute_Tags on $SourceString"); } my %MonthNumLibEn = ("01","Jan","02","Feb","03","Mar","04","Apr","05","May","06","Jun","07","Jul","08","Aug","09","Sep","10","Oct","11","Nov","12","Dec"); while ($SourceString =~ /%([ymdhwYMDHWNSO]+)-(\(\d+\)|\d+)/) { # Accept tag %xx-dd and %xx-(dd) my $timetag="$1"; my $timephase=quotemeta("$2"); my $timephasenb="$2"; $timephasenb=~s/[^\d]//g; if ($Debug) { debug(" Found a time tag '$timetag' with a phase of '$timephasenb' hour in log file name",1); } # Get older time my ($oldersec,$oldermin,$olderhour,$olderday,$oldermonth,$olderyear,$olderwday,$olderyday) = localtime($starttime-($timephasenb*3600)); my $olderweekofmonth=int($olderday/7); my $olderweekofyear=int(($olderyday-1+6-($olderwday==0?6:$olderwday-1))/7)+1; if ($olderweekofyear > 52) { $olderweekofyear = 1; } my $olderdaymod=$olderday%7; $olderwday++; my $olderns=Time::Local::timegm(0,0,0,$olderday,$oldermonth,$olderyear); if ($olderdaymod <= $olderwday) { if (($olderwday != 7) || ($olderdaymod != 0)) { $olderweekofmonth=$olderweekofmonth+1; } } if ($olderdaymod > $olderwday) { $olderweekofmonth=$olderweekofmonth+2; } # Change format of time variables $olderweekofmonth = "0$olderweekofmonth"; if ($olderweekofyear < 10) { $olderweekofyear = "0$olderweekofyear"; } if ($olderyear < 100) { $olderyear+=2000; } else { $olderyear+=1900; } my $oldersmallyear=$olderyear;$oldersmallyear =~ s/^..//; if (++$oldermonth < 10) { $oldermonth = "0$oldermonth"; } if ($olderday < 10) { $olderday = "0$olderday"; } if ($olderhour < 10) { $olderhour = "0$olderhour"; } if ($oldermin < 10) { $oldermin = "0$oldermin"; } if ($oldersec < 10) { $oldersec = "0$oldersec"; } # Replace tag with new value if ($timetag eq 'YYYY') { $SourceString =~ s/%YYYY-$timephase/$olderyear/ig; next; } if ($timetag eq 'YY') { $SourceString =~ s/%YY-$timephase/$oldersmallyear/ig; next; } if ($timetag eq 'MM') { $SourceString =~ s/%MM-$timephase/$oldermonth/ig; next; } if ($timetag eq 'MO') { $SourceString =~ s/%MO-$timephase/$MonthNumLibEn{$oldermonth}/ig; next; } if ($timetag eq 'DD') { $SourceString =~ s/%DD-$timephase/$olderday/ig; next; } if ($timetag eq 'HH') { $SourceString =~ s/%HH-$timephase/$olderhour/ig; next; } if ($timetag eq 'NS') { $SourceString =~ s/%NS-$timephase/$olderns/ig; next; } if ($timetag eq 'WM') { $SourceString =~ s/%WM-$timephase/$olderweekofmonth/g; next; } if ($timetag eq 'Wm') { my $olderweekofmonth0=$olderweekofmonth-1; $SourceString =~ s/%Wm-$timephase/$olderweekofmonth0/g; next; } if ($timetag eq 'WY') { $SourceString =~ s/%WY-$timephase/$olderweekofyear/g; next; } if ($timetag eq 'Wy') { my $olderweekofyear0=sprintf("%02d",$olderweekofyear-1); $SourceString =~ s/%Wy-$timephase/$olderweekofyear0/g; next; } if ($timetag eq 'DW') { $SourceString =~ s/%DW-$timephase/$olderwday/g; next; } if ($timetag eq 'Dw') { my $olderwday0=$olderwday-1; $SourceString =~ s/%Dw-$timephase/$olderwday0/g; next; } # If unknown tag error("Unknown tag '\%$timetag' in parameter."); } # Replace %YYYY %YY %MM %DD %HH with current value. Kept for backward compatibility. $SourceString =~ s/%YYYY/$nowyear/ig; $SourceString =~ s/%YY/$nowsmallyear/ig; $SourceString =~ s/%MM/$nowmonth/ig; $SourceString =~ s/%MO/$MonthNumLibEn{$nowmonth}/ig; $SourceString =~ s/%DD/$nowday/ig; $SourceString =~ s/%HH/$nowhour/ig; $SourceString =~ s/%NS/$nowns/ig; $SourceString =~ s/%WM/$nowweekofmonth/g; my $nowweekofmonth0=$nowweekofmonth-1; $SourceString =~ s/%Wm/$nowweekofmonth0/g; $SourceString =~ s/%WY/$nowweekofyear/g; my $nowweekofyear0=$nowweekofyear-1; $SourceString =~ s/%Wy/$nowweekofyear0/g; $SourceString =~ s/%DW/$nowwday/g; my $nowwday0=$nowwday-1; $SourceString =~ s/%Dw/$nowwday0/g; return $SourceString; } #------------------------------------------------------------------------------ # Function: Check if all parameters are correctly defined. If not set them to default. # Parameters: None # Input: All global variables # Output: Change on some global variables # Return: None #------------------------------------------------------------------------------ sub Check_Config { if ($Debug) { debug("Call to Check_Config"); } # Show initial values of main parameters before check if ($Debug) { debug(" LogFile='$LogFile'",2); debug(" LogType='$LogType'",2); debug(" LogFormat='$LogFormat'",2); debug(" LogSeparator='$LogSeparator'",2); debug(" DNSLookup='$DNSLookup'",2); debug(" DirData='$DirData'",2); debug(" DirCgi='$DirCgi'",2); debug(" DirIcons='$DirIcons'",2); debug(" NotPageList ".(join(',',keys %NotPageList)),2); debug(" ValidHTTPCodes ".(join(',',keys %ValidHTTPCodes)),2); debug(" ValidSMTPCodes ".(join(',',keys %ValidSMTPCodes)),2); debug(" UseFramesWhenCGI=$UseFramesWhenCGI",2); debug(" BuildReportFormat=$BuildReportFormat",2); debug(" BuildHistoryFormat=$BuildHistoryFormat",2); debug(" URLWithQueryWithOnlyFollowingParameters=".(join(',',@URLWithQueryWithOnly)),2); debug(" URLWithQueryWithoutFollowingParameters=".(join(',',@URLWithQueryWithout)),2); } # Main section $LogFile=&Substitute_Tags($LogFile); if (! $LogFile) { error("LogFile parameter is not defined in config/domain file"); } if ($LogType !~ /[WSMF]/i) { $LogType='W'; } $LogFormat =~ s/\\//g; if (! $LogFormat) { error("LogFormat parameter is not defined in config/domain file"); } if ($LogFormat =~ /^\d$/ && $LogFormat !~ /[1-6]/) { error("LogFormat parameter is wrong in config/domain file. Value is '$LogFormat' (should be 1,2,3,4,5 or a 'personalized AWStats log format string')"); } $LogSeparator||="\\s"; $DirData||='.'; $DirCgi||='/cgi-bin'; $DirIcons||='/icon'; if ($DNSLookup !~ /[0-2]/) { error("DNSLookup parameter is wrong in config/domain file. Value is '$DNSLookup' (should be 0,1 or 2)"); } if (! $SiteDomain) { error("SiteDomain parameter not defined in your config/domain file. You must edit it for using this version of AWStats."); } if ($AllowToUpdateStatsFromBrowser !~ /[0-1]/) { $AllowToUpdateStatsFromBrowser=0; } if ($AllowFullYearView !~ /[0-3]/) { $AllowFullYearView=2; } # Optional setup section if ($EnableLockForUpdate !~ /[0-1]/) { $EnableLockForUpdate=0; } $DNSStaticCacheFile||='dnscache.txt'; $DNSLastUpdateCacheFile||='dnscachelastupdate.txt'; if ($DNSStaticCacheFile eq $DNSLastUpdateCacheFile) { error("DNSStaticCacheFile and DNSLastUpdateCacheFile must have different values."); } if ($AllowAccessFromWebToAuthenticatedUsersOnly !~ /[0-1]/) { $AllowAccessFromWebToAuthenticatedUsersOnly=0; } if ($CreateDirDataIfNotExists !~ /[0-1]/) { $CreateDirDataIfNotExists=0; } if ($BuildReportFormat !~ /html|xhtml|xml/i) { $BuildReportFormat='html'; } if ($BuildHistoryFormat !~ /text|xml/) { $BuildHistoryFormat='text'; } if ($SaveDatabaseFilesWithPermissionsForEveryone !~ /[0-1]/) { $SaveDatabaseFilesWithPermissionsForEveryone=0; } if ($PurgeLogFile !~ /[0-1]/) { $PurgeLogFile=0; } if ($KeepBackupOfHistoricFiles !~ /[0-1]/) { $KeepBackupOfHistoricFiles=0; } $DefaultFile[0]||='index.html'; if ($AuthenticatedUsersNotCaseSensitive !~ /[0-1]/) { $AuthenticatedUsersNotCaseSensitive=0; } if ($URLNotCaseSensitive !~ /[0-1]/) { $URLNotCaseSensitive=0; } if ($URLWithAnchor !~ /[0-1]/) { $URLWithAnchor=0; } $URLQuerySeparators =~ s/\s//g; if (! $URLQuerySeparators) { $URLQuerySeparators='?;'; } if ($URLWithQuery !~ /[0-1]/) { $URLWithQuery=0; } if ($URLReferrerWithQuery !~ /[0-1]/) { $URLReferrerWithQuery=0; } if ($WarningMessages !~ /[0-1]/) { $WarningMessages=1; } if ($DebugMessages !~ /[0-1]/) { $DebugMessages=0; } if ($NbOfLinesForCorruptedLog !~ /^\d+/ || $NbOfLinesForCorruptedLog<1) { $NbOfLinesForCorruptedLog=50; } if ($Expires !~ /^\d+/) { $Expires=0; } if ($DecodeUA !~ /[0-1]/) { $DecodeUA=0; } $MiscTrackerUrl||='/js/awstats_misc_tracker.js'; # Optional accuracy setup section if ($LevelForWormsDetection !~ /^\d+/) { $LevelForWormsDetection=0; } if ($LevelForRobotsDetection !~ /^\d+/) { $LevelForRobotsDetection=2; } if ($LevelForBrowsersDetection !~ /^\d+/) { $LevelForBrowsersDetection=2; } if ($LevelForOSDetection !~ /^\d+/) { $LevelForOSDetection=2; } if ($LevelForRefererAnalyze !~ /^\d+/) { $LevelForRefererAnalyze=2; } if ($LevelForFileTypesDetection !~ /^\d+/) { $LevelForFileTypesDetection=2; } if ($LevelForSearchEnginesDetection !~ /^\d+/) { $LevelForSearchEnginesDetection=2; } if ($LevelForKeywordsDetection !~ /^\d+/) { $LevelForKeywordsDetection=2; } # Optional extra setup section foreach my $extracpt (1..@ExtraName-1) { if ($ExtraStatTypes[$extracpt] !~ /[PHBL]/) { $ExtraStatTypes[$extracpt]='PHBL'; } if ($MaxNbOfExtra[$extracpt] !~ /^\d+$/ || $MaxNbOfExtra[$extracpt]<1) { $MaxNbOfExtra[$extracpt]=20; } if ($MinHitExtra[$extracpt] !~ /^\d+$/ || $MinHitExtra[$extracpt]<1) { $MinHitExtra[$extracpt]=1; } if (! $ExtraFirstColumnValues[$extracpt]) { error("Extra section number $extracpt is defined without ExtraSectionFirstColumnValues$extracpt parameter"); } if (! $ExtraFirstColumnFormat[$extracpt]) { $ExtraFirstColumnFormat[$extracpt] = '%s'; } } # Optional appearance setup section if ($MaxRowsInHTMLOutput !~ /^\d+/ || $MaxRowsInHTMLOutput<1) { $MaxRowsInHTMLOutput=1000; } if ($ShowMenu !~ /[01]/) { $ShowMenu=1; } if ($ShowSummary !~ /[01UVPHB]/) { $ShowSummary='UVPHB'; } if ($ShowMonthStats !~ /[01UVPHB]/) { $ShowMonthStats='UVPHB'; } if ($ShowDaysOfMonthStats !~ /[01VPHB]/) { $ShowDaysOfMonthStats='VPHB'; } if ($ShowDaysOfWeekStats !~ /[01PHBL]/) { $ShowDaysOfWeekStats='PHBL'; } if ($ShowHoursStats !~ /[01PHBL]/) { $ShowHoursStats='PHBL'; } if ($ShowDomainsStats !~ /[01PHB]/) { $ShowDomainsStats='PHB'; } if ($ShowHostsStats !~ /[01PHBL]/) { $ShowHostsStats='PHBL'; } if ($ShowAuthenticatedUsers !~ /[01PHBL]/) { $ShowAuthenticatedUsers=0; } if ($ShowRobotsStats !~ /[01HBL]/) { $ShowRobotsStats='HBL'; } if ($ShowWormsStats !~ /[01HBL]/) { $ShowWormsStats='HBL'; } if ($ShowEMailSenders !~ /[01HBML]/) { $ShowEMailSenders=0; } if ($ShowEMailReceivers !~ /[01HBML]/) { $ShowEMailReceivers=0; } if ($ShowSessionsStats !~ /[01]/) { $ShowSessionsStats=1; } if ($ShowPagesStats !~ /[01PBEX]/i) { $ShowPagesStats='PBEX'; } if ($ShowFileTypesStats !~ /[01HBC]/) { $ShowFileTypesStats='HB'; } if ($ShowFileSizesStats !~ /[01]/) { $ShowFileSizesStats=1; } if ($ShowOSStats !~ /[01]/) { $ShowOSStats=1; } if ($ShowBrowsersStats !~ /[01]/) { $ShowBrowsersStats=1; } if ($ShowScreenSizeStats !~ /[01]/) { $ShowScreenSizeStats=0; } if ($ShowOriginStats !~ /[01PH]/) { $ShowOriginStats='PH'; } if ($ShowKeyphrasesStats !~ /[01]/) { $ShowKeyphrasesStats=1; } if ($ShowKeywordsStats !~ /[01]/) { $ShowKeywordsStats=1; } if ($ShowClusterStats !~ /[01PHB]/) { $ShowClusterStats=0; } if ($ShowMiscStats !~ /[01anjdfrqwp]/) { $ShowMiscStats='a'; } if ($ShowHTTPErrorsStats !~ /[01]/) { $ShowHTTPErrorsStats=1; } if ($ShowSMTPErrorsStats !~ /[01]/) { $ShowSMTPErrorsStats=0; } if ($AddDataArrayMonthStats !~ /[01]/) { $AddDataArrayMonthStats=1; } if ($AddDataArrayShowDaysOfMonthStats !~ /[01]/) { $AddDataArrayShowDaysOfMonthStats=1; } if ($AddDataArrayShowDaysOfWeekStats !~ /[01]/) { $AddDataArrayShowDaysOfWeekStats=1; } if ($AddDataArrayShowHoursStats !~ /[01]/) { $AddDataArrayShowHoursStats=1; } my @maxnboflist=('Domain','HostsShown','LoginShown','RobotShown','WormsShown','PageShown','OsShown','BrowsersShown','ScreenSizesShown','RefererShown','KeyphrasesShown','KeywordsShown','EMailsShown'); my @maxnboflistdefaultval=(10,10,10,10,5,10,10,10,5,10,10,10,20); foreach my $i (0..(@maxnboflist-1)) { if (! $MaxNbOf{$maxnboflist[$i]} || $MaxNbOf{$maxnboflist[$i]} !~ /^\d+$/ || $MaxNbOf{$maxnboflist[$i]}<1) { $MaxNbOf{$maxnboflist[$i]}=$maxnboflistdefaultval[$i]; } } my @minhitlist=('Domain','Host','Login','Robot','Worm','File','Os','Browser','ScreenSize','Refer','Keyphrase','Keyword','EMail'); my @minhitlistdefaultval=(1,1,1,1,1,1,1,1,1,1,1,1,1); foreach my $i (0..(@minhitlist-1)) { if (! $MinHit{$minhitlist[$i]} || $MinHit{$minhitlist[$i]} !~ /^\d+$/ || $MinHit{$minhitlist[$i]}<1) { $MinHit{$minhitlist[$i]}=$minhitlistdefaultval[$i]; } } if ($FirstDayOfWeek !~ /[01]/) { $FirstDayOfWeek=1; } if ($UseFramesWhenCGI !~ /[01]/) { $UseFramesWhenCGI=1; } if ($DetailedReportsOnNewWindows !~ /[012]/) { $DetailedReportsOnNewWindows=1; } if ($ShowLinksOnUrl !~ /[01]/) { $ShowLinksOnUrl=1; } if ($MaxLengthOfShownURL !~ /^\d+/ || $MaxLengthOfShownURL<1) { $MaxLengthOfShownURL=64; } if ($ShowLinksToWhoIs !~ /[01]/) { $ShowLinksToWhoIs=0; } $Logo||='awstats_logo6.png'; $LogoLink||='http://awstats.sourceforge.net'; if ($BarWidth !~ /^\d+/ || $BarWidth<1) { $BarWidth=260; } if ($BarHeight !~ /^\d+/ || $BarHeight<1) { $BarHeight=90; } $color_Background =~ s/#//g; if ($color_Background !~ /^[0-9|A-H]+$/i) { $color_Background='FFFFFF'; } $color_TableBGTitle =~ s/#//g; if ($color_TableBGTitle !~ /^[0-9|A-H]+$/i) { $color_TableBGTitle='CCCCDD'; } $color_TableTitle =~ s/#//g; if ($color_TableTitle !~ /^[0-9|A-H]+$/i) { $color_TableTitle='000000'; } $color_TableBG =~ s/#//g; if ($color_TableBG !~ /^[0-9|A-H]+$/i) { $color_TableBG='CCCCDD'; } $color_TableRowTitle =~ s/#//g; if ($color_TableRowTitle !~ /^[0-9|A-H]+$/i) { $color_TableRowTitle='FFFFFF'; } $color_TableBGRowTitle =~ s/#//g; if ($color_TableBGRowTitle !~ /^[0-9|A-H]+$/i) { $color_TableBGRowTitle='ECECEC'; } $color_TableBorder =~ s/#//g; if ($color_TableBorder !~ /^[0-9|A-H]+$/i) { $color_TableBorder='ECECEC'; } $color_text =~ s/#//g; if ($color_text !~ /^[0-9|A-H]+$/i) { $color_text='000000'; } $color_textpercent =~ s/#//g; if ($color_textpercent !~ /^[0-9|A-H]+$/i) { $color_textpercent='606060'; } $color_titletext =~ s/#//g; if ($color_titletext !~ /^[0-9|A-H]+$/i) { $color_titletext='000000'; } $color_weekend =~ s/#//g; if ($color_weekend !~ /^[0-9|A-H]+$/i) { $color_weekend='EAEAEA'; } $color_link =~ s/#//g; if ($color_link !~ /^[0-9|A-H]+$/i) { $color_link='0011BB'; } $color_hover =~ s/#//g; if ($color_hover !~ /^[0-9|A-H]+$/i) { $color_hover='605040'; } $color_other =~ s/#//g; if ($color_other !~ /^[0-9|A-H]+$/i) { $color_other='666688'; } $color_u =~ s/#//g; if ($color_u !~ /^[0-9|A-H]+$/i) { $color_u='FFA060'; } $color_v =~ s/#//g; if ($color_v !~ /^[0-9|A-H]+$/i) { $color_v='F4F090'; } $color_p =~ s/#//g; if ($color_p !~ /^[0-9|A-H]+$/i) { $color_p='4477DD'; } $color_h =~ s/#//g; if ($color_h !~ /^[0-9|A-H]+$/i) { $color_h='66EEFF'; } $color_k =~ s/#//g; if ($color_k !~ /^[0-9|A-H]+$/i) { $color_k='2EA495'; } $color_s =~ s/#//g; if ($color_s !~ /^[0-9|A-H]+$/i) { $color_s='8888DD'; } $color_e =~ s/#//g; if ($color_e !~ /^[0-9|A-H]+$/i) { $color_e='CEC2E8'; } $color_x =~ s/#//g; if ($color_x !~ /^[0-9|A-H]+$/i) { $color_x='C1B2E2'; } # Correct param if default value is asked if ($ShowSummary eq '1') { $ShowSummary = 'UVPHB'; } if ($ShowMonthStats eq '1') { $ShowMonthStats = 'UVPHB'; } if ($ShowDaysOfMonthStats eq '1') { $ShowDaysOfMonthStats = 'VPHB'; } if ($ShowDaysOfWeekStats eq '1') { $ShowDaysOfWeekStats = 'PHBL'; } if ($ShowHoursStats eq '1') { $ShowHoursStats = 'PHBL'; } if ($ShowDomainsStats eq '1') { $ShowDomainsStats = 'PHB'; } if ($ShowHostsStats eq '1') { $ShowHostsStats = 'PHBL'; } if ($ShowEMailSenders eq '1') { $ShowEMailSenders = 'HBML'; } if ($ShowEMailReceivers eq '1') { $ShowEMailReceivers = 'HBML'; } if ($ShowAuthenticatedUsers eq '1') { $ShowAuthenticatedUsers = 'PHBL'; } if ($ShowRobotsStats eq '1') { $ShowRobotsStats = 'HBL'; } if ($ShowWormsStats eq '1') { $ShowWormsStats = 'HBL'; } if ($ShowPagesStats eq '1') { $ShowPagesStats = 'PBEX'; } if ($ShowFileTypesStats eq '1') { $ShowFileTypesStats = 'HB'; } if ($ShowOriginStats eq '1') { $ShowOriginStats = 'PH'; } if ($ShowClusterStats eq '1') { $ShowClusterStats = 'PHB'; } if ($ShowMiscStats eq '1') { $ShowMiscStats = 'anjdfrqwp'; } # Convert extra sections data into @ExtraConditionType, @ExtraConditionTypeVal... foreach my $extranum (1..@ExtraName-1) { my $part=0; foreach my $conditioncouple (split(/\s*\|\|\s*/, $ExtraCondition[$extranum])) { my ($conditiontype, $conditiontypeval)=split(/[,:]/,$conditioncouple,2); $ExtraConditionType[$extranum][$part]=$conditiontype; if ($conditiontypeval =~ /^REGEX\[(.*)\]$/i) { $conditiontypeval=$1; } #else { $conditiontypeval=quotemeta($conditiontypeval); } $ExtraConditionTypeVal[$extranum][$part]=qr/$conditiontypeval/i; $part++; } $part=0; foreach my $rowkeycouple (split(/\s*\|\|\s*/, $ExtraFirstColumnValues[$extranum])) { my ($rowkeytype, $rowkeytypeval)=split(/[,:]/,$rowkeycouple,2); $ExtraFirstColumnValuesType[$extranum][$part]=$rowkeytype; if ($rowkeytypeval =~ /^REGEX\[(.*)\]$/i) { $rowkeytypeval=$1; } #else { $rowkeytypeval=quotemeta($rowkeytypeval); } $ExtraFirstColumnValuesTypeVal[$extranum][$part]=qr/$rowkeytypeval/i; $part++; } } # Show definitive value for major parameters if ($Debug) { debug(" LogFile='$LogFile'",2); debug(" LogFormat='$LogFormat'",2); debug(" LogSeparator='$LogSeparator'",2); debug(" DNSLookup='$DNSLookup'",2); debug(" DirData='$DirData'",2); debug(" DirCgi='$DirCgi'",2); debug(" DirIcons='$DirIcons'",2); debug(" SiteDomain='$SiteDomain'",2); debug(" MiscTrackerUrl='$MiscTrackerUrl'",2); foreach (keys %MaxNbOf) { debug(" MaxNbOf{$_}=$MaxNbOf{$_}",2); } foreach (keys %MinHit) { debug(" MinHit{$_}=$MinHit{$_}",2); } foreach my $extranum (1..@ExtraName-1) { debug(" ExtraCodeFilter[$extranum] is array ".join(',',@{$ExtraCodeFilter[$extranum]}),2); debug(" ExtraConditionType[$extranum] is array ".join(',',@{$ExtraConditionType[$extranum]}),2); debug(" ExtraConditionTypeVal[$extranum] is array ".join(',',@{$ExtraConditionTypeVal[$extranum]}),2); debug(" ExtraFirstColumnValuesType[$extranum] is array ".join(',',@{$ExtraFirstColumnValuesType[$extranum]}),2); debug(" ExtraFirstColumnValuesTypeVal[$extranum] is array ".join(',',@{$ExtraFirstColumnValuesTypeVal[$extranum]}),2); } } # Deny URLWithQueryWithOnlyFollowingParameters and URLWithQueryWithoutFollowingParameters both set if (@URLWithQueryWithOnly && @URLWithQueryWithout) { error("URLWithQueryWithOnlyFollowingParameters and URLWithQueryWithoutFollowingParameters can't be both set at the same time"); } # Deny $ShowHTTPErrorsStats and $ShowSMTPErrorsStats both set if ($ShowHTTPErrorsStats && $ShowSMTPErrorsStats) { error("ShowHTTPErrorsStats and ShowSMTPErrorsStats can't be both set at the same time"); } # Deny LogFile if contains a pipe and PurgeLogFile || ArchiveLogRecords set on if (($PurgeLogFile || $ArchiveLogRecords) && $LogFile =~ /\|\s*$/) { error("A pipe in log file name is not allowed if PurgeLogFile and ArchiveLogRecords are not set to 0"); } # If not a migrate, check if DirData is OK if (! $MigrateStats && ! -d $DirData) { if ($CreateDirDataIfNotExists) { if ($Debug) { debug(" Make directory $DirData",2); } my $mkdirok=mkdir "$DirData", 0766; if (! $mkdirok) { error("$PROG failed to create directory DirData (DirData=\"$DirData\", CreateDirDataIfNotExists=$CreateDirDataIfNotExists)."); } } else { error("AWStats database directory defined in config file by 'DirData' parameter ($DirData) does not exist or is not writable."); } } if ($LogType eq 'S') { $NOTSORTEDRECORDTOLERANCE=1000000; } } #------------------------------------------------------------------------------ # Function: Common function used by init function of plugins # Parameters: AWStats version required by plugin # Input: $VERSION # Output: None # Return: '' if ok, "Error: xxx" if error #------------------------------------------------------------------------------ sub Check_Plugin_Version { my $PluginNeedAWStatsVersion=shift; if (! $PluginNeedAWStatsVersion) { return 0; } $VERSION =~ /^(\d+)\.(\d+)/; my $numAWStatsVersion=($1*1000)+$2; $PluginNeedAWStatsVersion =~ /^(\d+)\.(\d+)/; my $numPluginNeedAWStatsVersion=($1*1000)+$2; if ($numPluginNeedAWStatsVersion > $numAWStatsVersion) { return "Error: AWStats version $PluginNeedAWStatsVersion or higher is required. Detected $VERSION."; } return ''; } #------------------------------------------------------------------------------ # Function: Return a checksum for an array of string # Parameters: Array of string # Input: None # Output: None # Return: Checksum number #------------------------------------------------------------------------------ sub CheckSum { my $string=shift; my $checksum=0; # use MD5; # $checksum = MD5->hexhash($string); my $i=0; my $j=0; while ($i < length($string)) { my $c=substr($string,$i,1); $checksum+=(ord($c)<<(8*$j)); if ($j++ > 3) { $j=0; } $i++; } return $checksum; } #------------------------------------------------------------------------------ # Function: Load plugins files # Parameters: None # Input: $DIR @PluginsToLoad # Output: None # Return: None #------------------------------------------------------------------------------ sub Read_Plugins { # Check plugin files in common possible directories : # Windows and standard package: "$DIR/plugins" (plugins in same dir than awstats.pl) # Redhat : "/usr/local/awstats/wwwroot/cgi-bin/plugins" # Debian package : "/usr/share/awstats/plugins" my @PossiblePluginsDir=("$DIR/plugins","/usr/local/awstats/wwwroot/cgi-bin/plugins","/usr/share/awstats/plugins"); my %DirAddedInINC=(); #Removed for security reason #foreach my $key (keys %NoLoadPlugin) { if ($NoLoadPlugin{$key} < 0) { push @PluginsToLoad, $key; } } if ($Debug) { debug("Call to Read_Plugins with list: ".join(',',@PluginsToLoad)); } foreach my $plugininfo (@PluginsToLoad) { my ($pluginfile,$pluginparam)=split(/\s+/,$plugininfo,2); $pluginfile =~ s/\.pm$//i; $pluginfile =~ /([^\/\\]+)$/; my $pluginname=$1; # pluginname is pluginfile without any path # Check if plugin is not disabled if ($NoLoadPlugin{$pluginname} && $NoLoadPlugin{$pluginname} > 0) { if ($Debug) { debug(" Plugin load for '$pluginfile' has been disabled from parameters"); } next; } if ($pluginname) { if (! $PluginsLoaded{'init'}{"$pluginname"}) { # Plugin not already loaded my %pluginisfor=('timehires'=>'u','ipv6'=>'u','hashfiles'=>'u', 'geoipfree'=>'u', 'geoip'=>'ou','geoip_region_maxmind'=>'mou','geoip_city_maxmind'=>'mou','geoip_isp_maxmind'=>'mou','geoip_org_maxmind'=>'mou', 'timezone'=>'ou', 'decodeutfkeys'=>'o','hostinfo'=>'o','rawlog'=>'o','userinfo'=>'o','urlalias'=>'o','tooltips'=>'o'); if ($pluginisfor{$pluginname}) { # If it's a known plugin, may be we don't need to load it # Do not load "menu handler plugins" if output only and mainleft frame if (! $UpdateStats && scalar keys %HTMLOutput && $FrameName eq 'mainleft' && $pluginisfor{$pluginname} !~ /m/) { $PluginsLoaded{'init'}{"$pluginname"}=1; next; } # Do not load "update plugins" if output only if (! $UpdateStats && scalar keys %HTMLOutput && $pluginisfor{$pluginname} !~ /o/) { $PluginsLoaded{'init'}{"$pluginname"}=1; next; } # Do not load "output plugins" if update only if ($UpdateStats && ! scalar keys %HTMLOutput && $pluginisfor{$pluginname} !~ /u/) { $PluginsLoaded{'init'}{"$pluginname"}=1; next; } } # Load plugin foreach my $dir (@PossiblePluginsDir) { my $searchdir=$dir; if ($searchdir && (!($searchdir =~ /\/$/)) && (!($searchdir =~ /\\$/)) ) { $searchdir .= "/"; } my $pluginpath="${searchdir}${pluginfile}.pm"; if (-s "$pluginpath") { $PluginDir="${searchdir}"; # Set plugin dir if ($Debug) { debug(" Try to init plugin '$pluginname' ($pluginpath) with param '$pluginparam'",1); } if (! $DirAddedInINC{"$dir"}) { push @INC, "$dir"; $DirAddedInINC{"$dir"}=1; } my $loadret=0; my $modperl=$ENV{"MOD_PERL"}? eval { require mod_perl; $mod_perl::VERSION >= 1.99 ? 2 : 1 } : 0; if ($modperl == 2) { $loadret=require "$pluginpath"; } else { $loadret=require "$pluginfile.pm"; } if (! $loadret || $loadret =~ /^error/i) { # Load failed, we stop here error("Plugin load for plugin '$pluginname' failed with return code: $loadret"); } my $ret; # To get init return my $initfunction="\$ret=Init_$pluginname('$pluginparam')"; my $initret=eval("$initfunction"); if ($initret eq 'xxx') { $initret='Error: The PluginHooksFunctions variable defined in plugin file does not contain list of hooked functions'; } if (! $initret || $initret =~ /^error/i) { # Init function failed, we stop here error("Plugin init for plugin '$pluginname' failed with return code: ".($initret?"$initret":"$@ (A module required by plugin might be missing).")); } # Plugin load and init successfull foreach my $elem (split(/\s+/,$initret)) { # Some functions can only be plugged once my @uniquefunc=('GetCountryCodeByName','GetCountryCodeByAddr','ChangeTime','GetTimeZoneTitle','GetTime','SearchFile','LoadCache','SaveHash','ShowMenu'); my $isuniquefunc=0; foreach my $function (@uniquefunc) { if ("$elem" eq "$function") { # We try to load a 'unique' function, so we check and stop if already loaded foreach my $otherpluginname (keys %{$PluginsLoaded{"$elem"}}) { error("Conflict between plugin '$pluginname' and '$otherpluginname'. They both implements the 'must be unique' function '$elem'.\nYou must choose between one of them. Using together is not possible."); } $isuniquefunc=1; last; } } if ($isuniquefunc) { # TODO Use $PluginsLoaded{"$elem"}="$pluginname"; for unique func $PluginsLoaded{"$elem"}{"$pluginname"}=1; } else { $PluginsLoaded{"$elem"}{"$pluginname"}=1; } if ("$elem" =~ /SectionInitHashArray/) { $AtLeastOneSectionPlugin=1; } } $PluginsLoaded{'init'}{"$pluginname"}=1; if ($Debug) { debug(" Plugin '$pluginname' now hooks functions '$initret'",1); } last; } } if (! $PluginsLoaded{'init'}{"$pluginname"}) { error("AWStats config file contains a directive to load plugin \"$pluginname\" (LoadPlugin=\"$plugininfo\") but AWStats can't open plugin file \"$pluginfile.pm\" for read.\nCheck if file is in \"".($PossiblePluginsDir[0])."\" directory and is readable."); } } else { warning("Warning: Tried to load plugin \"$pluginname\" twice. Fix config file."); } } else { error("Plugin \"$pluginfile\" is not a valid plugin name."); } } # In output mode, geo ip plugins are not loaded, so message changes are done here (can't be done in plugin init function) if ($PluginsLoaded{'init'}{'geoip'} || $PluginsLoaded{'init'}{'geoipfree'}) { $Message[17]=$Message[25]=$Message[148]; } } #------------------------------------------------------------------------------ # Function: Read history file and create/update tmp history file # Parameters: year,month,withupdate,withpurge,part_to_load[,lastlinenb,lastlineoffset,lastlinechecksum] # Input: $DirData $PROG $FileSuffix $LastLine # Output: None # Return: Tmp history file name created/updated or '' if withupdate is 0 #------------------------------------------------------------------------------ sub Read_History_With_TmpUpdate { my $year=sprintf("%04i",shift||0); my $month=sprintf("%02i",shift||0); my $withupdate=shift||0; my $withpurge=shift||0; my $part=shift||''; my $xml=($BuildHistoryFormat eq 'xml'?1:0); my $xmleb=''; my $xmlrb=''; my $lastlinenb=shift||0; my $lastlineoffset=shift||0; my $lastlinechecksum=shift||0; my %allsections=('general'=>1,'misc'=>2,'time'=>3,'visitor'=>4,'day'=>5, 'domain'=>6,'cluster'=>7,'login'=>8,'robot'=>9,'worms'=>10,'emailsender'=>11,'emailreceiver'=>12, 'session'=>13,'sider'=>14,'filetypes'=>15, 'os'=>16,'browser'=>17,'screensize'=>18,'unknownreferer'=>19,'unknownrefererbrowser'=>20, 'origin'=>21,'sereferrals'=>22,'pagerefs'=>23, 'searchwords'=>24,'keywords'=>25, 'errors'=>26); my $order=(scalar keys %allsections)+1; foreach (keys %TrapInfosForHTTPErrorCodes) { $allsections{"sider_$_"}=$order++; } foreach (1..@ExtraName-1) { $allsections{"extra_$_"}=$order++; } foreach (keys %{$PluginsLoaded{'SectionInitHashArray'}}) { $allsections{"plugin_$_"}=$order++; } my $withread=0; # Variable used to read old format history files my $readvisitorforbackward=0; # In standard use of AWStats, the DayRequired variable is always empty if ($DayRequired) { if ($Debug) { debug("Call to Read_History_With_TmpUpdate [$year,$month,withupdate=$withupdate,withpurge=$withpurge,part=$part,lastlinenb=$lastlinenb,lastlineoffset=$lastlineoffset,lastlinechecksum=$lastlinechecksum] ($DayRequired)"); } } else { if ($Debug) { debug("Call to Read_History_With_TmpUpdate [$year,$month,withupdate=$withupdate,withpurge=$withpurge,part=$part,lastlinenb=$lastlinenb,lastlineoffset=$lastlineoffset,lastlinechecksum=$lastlinechecksum]"); } } # Define SectionsToLoad (which sections to load) my %SectionsToLoad = (); if ($part eq 'all') { # Load all needed sections my $order=1; $SectionsToLoad{'general'}=$order++; # When $SectionsToLoad{'time'}=$order++; # Always loaded because needed to count TotalPages, TotalHits, TotalBandwidth if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowHostsStats) || $HTMLOutput{'allhosts'} || $HTMLOutput{'lasthosts'} || $HTMLOutput{'unknownip'}) { $SectionsToLoad{'visitor'}=$order++; } # Must be before day, sider and session section if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && ($ShowDaysOfWeekStats || $ShowDaysOfMonthStats)) || $HTMLOutput{'alldays'}) { $SectionsToLoad{'day'}=$order++; } # Who if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowDomainsStats) || $HTMLOutput{'alldomains'}) { $SectionsToLoad{'domain'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowAuthenticatedUsers) || $HTMLOutput{'alllogins'} || $HTMLOutput{'lastlogins'}) { $SectionsToLoad{'login'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowRobotsStats) || $HTMLOutput{'allrobots'} || $HTMLOutput{'lastrobots'}) { $SectionsToLoad{'robot'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowWormsStats) || $HTMLOutput{'allworms'} || $HTMLOutput{'lastworms'}) { $SectionsToLoad{'worms'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowEMailSenders) || $HTMLOutput{'allemails'} || $HTMLOutput{'lastemails'}) { $SectionsToLoad{'emailsender'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowEMailReceivers) || $HTMLOutput{'allemailr'} || $HTMLOutput{'lastemailr'}) { $SectionsToLoad{'emailreceiver'}=$order++; } # Navigation if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowSessionsStats) || $HTMLOutput{'sessions'}) { $SectionsToLoad{'session'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowPagesStats) || $HTMLOutput{'urldetail'} || $HTMLOutput{'urlentry'} || $HTMLOutput{'urlexit'}) { $SectionsToLoad{'sider'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowFileTypesStats) || $HTMLOutput{'filetypes'}) { $SectionsToLoad{'filetypes'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowOSStats) || $HTMLOutput{'osdetail'}) { $SectionsToLoad{'os'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowBrowsersStats) || $HTMLOutput{'browserdetail'}) { $SectionsToLoad{'browser'}=$order++; } if ($UpdateStats || $MigrateStats || $HTMLOutput{'unknownos'}) { $SectionsToLoad{'unknownreferer'}=$order++; } if ($UpdateStats || $MigrateStats || $HTMLOutput{'unknownbrowser'}) { $SectionsToLoad{'unknownrefererbrowser'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowScreenSizeStats)) { $SectionsToLoad{'screensize'}=$order++; } # Referers if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowOriginStats) || $HTMLOutput{'origin'}) { $SectionsToLoad{'origin'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowOriginStats) || $HTMLOutput{'refererse'}) { $SectionsToLoad{'sereferrals'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowOriginStats) || $HTMLOutput{'refererpages'}) { $SectionsToLoad{'pagerefs'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowKeyphrasesStats) || $HTMLOutput{'keyphrases'} || $HTMLOutput{'keywords'}) { $SectionsToLoad{'searchwords'}=$order++; } if (! $withupdate && $HTMLOutput{'main'} && $ShowKeywordsStats) { $SectionsToLoad{'keywords'}=$order++; } # If we update, dont need to load # Others if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowMiscStats)) { $SectionsToLoad{'misc'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && ($ShowHTTPErrorsStats || $ShowSMTPErrorsStats)) || $HTMLOutput{'errors'}) { $SectionsToLoad{'errors'}=$order++; } foreach (keys %TrapInfosForHTTPErrorCodes) { if ($UpdateStats || $MigrateStats || $HTMLOutput{"errors$_"}) { $SectionsToLoad{"sider_$_"}=$order++; } } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowClusterStats)) { $SectionsToLoad{'cluster'}=$order++; } foreach (1..@ExtraName-1) { if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ExtraStatTypes[$_]) || $HTMLOutput{"extra$_"}) { $SectionsToLoad{"extra_$_"}=$order++; } } foreach (keys %{$PluginsLoaded{'SectionInitHashArray'}}) { if ($UpdateStats || $MigrateStats || $HTMLOutput{"plugin_$_"}) { $SectionsToLoad{"plugin_$_"}=$order++; } } } else { # Load only required sections my $order=1; foreach (split(/\s+/,$part)) { $SectionsToLoad{$_}=$order++; } } # Define SectionsToSave (which sections to save) my %SectionsToSave = (); if ($withupdate) { %SectionsToSave=%allsections; } if ($Debug) { debug(" List of sections marked for load : ".join(' ',(sort { $SectionsToLoad{$a} <=> $SectionsToLoad{$b} } keys %SectionsToLoad)),2); debug(" List of sections marked for save : ".join(' ',(sort { $SectionsToSave{$a} <=> $SectionsToSave{$b} } keys %SectionsToSave)),2); } # Define value for filetowrite and filetoread (Month before Year kept for backward compatibility) my $filetowrite=''; my $filetoread=''; if ($HistoryAlreadyFlushed{"$year$month"} && -s "$DirData/$PROG$month$year$FileSuffix.tmp.$$") { # tmp history file was already flushed $filetoread="$DirData/$PROG$month$year$FileSuffix.tmp.$$"; $filetowrite="$DirData/$PROG$month$year$FileSuffix.tmp.$$.bis"; } else { $filetoread="$DirData/$PROG$DayRequired$month$year$FileSuffix.txt"; $filetowrite="$DirData/$PROG$month$year$FileSuffix.tmp.$$"; } if ($Debug) { debug(" History file to read is '$filetoread'",2); } # Is there an old data file to read or, if migrate, can we open the file for read if (-s $filetoread || $MigrateStats) { $withread=1; } # Open files if ($withread) { open(HISTORY,$filetoread) || error("Couldn't open file \"$filetoread\" for read: $!","","",$MigrateStats); binmode HISTORY; # Avoid premature EOF due to history files corrupted with \cZ or bin chars } if ($withupdate) { open(HISTORYTMP,">$filetowrite") || error("Couldn't open file \"$filetowrite\" for write: $!"); binmode HISTORYTMP; if ($xml) { print HISTORYTMP "\n\n"; } Save_History("header",$year,$month); } # Loop on read file my $readxml=0; if ($withread) { my $countlines=0; my $versionnum=0; my @field=(); while () { chomp $_; s/\r//; $countlines++; # Test if it's xml if (! $readxml && $_ =~ /^ int($field[1])) { $FirstTime{$year.$month}=int($field[1]); }; next; } if ($field[0] eq 'LastTime' || $field[0] eq "${xmlrb}LastTime") { if (! $LastTime{$year.$month} || $LastTime{$year.$month} < int($field[1])) { $LastTime{$year.$month}=int($field[1]); }; next; } if ($field[0] eq 'LastUpdate' || $field[0] eq "${xmlrb}LastUpdate") { if ($LastUpdate < $field[1]) { $LastUpdate=int($field[1]); #$LastUpdateLinesRead=int($field[2]); #$LastUpdateNewLinesRead=int($field[3]); #$LastUpdateLinesCorrupted=int($field[4]); }; next; } if ($field[0] eq 'TotalVisits' || $field[0] eq "${xmlrb}TotalVisits") { if (! $withupdate) { $MonthVisits{$year.$month}+=int($field[1]); } # Save in MonthVisits also if migrate from a file < 4.x for backward compatibility if ($MigrateStats && $versionnum < 4000 && ! $MonthVisits{$year.$month}) { if ($Debug) { debug("File is version < 4000. We save ".int($field[1])." visits in DayXxx arrays",1); } $DayHits{$year.$month."00"}+=0; $DayVisits{$year.$month."00"}+=int($field[1]); } next; } if ($field[0] eq 'TotalUnique' || $field[0] eq "${xmlrb}TotalUnique") { if (! $withupdate) { $MonthUnique{$year.$month}+=int($field[1]); } next; } if ($field[0] eq 'MonthHostsKnown' || $field[0] eq "${xmlrb}MonthHostsKnown") { if (! $withupdate) { $MonthHostsKnown{$year.$month}+=int($field[1]); } next; } if ($field[0] eq 'MonthHostsUnknown' || $field[0] eq "${xmlrb}MonthHostsUnknown") { if (! $withupdate) { $MonthHostsUnknown{$year.$month}+=int($field[1]); } next; } if (($field[0] eq 'END_GENERAL' || $field[0] eq "${xmleb}END_GENERAL") # END_GENERAL didn't exist for history files < 5.0 || ($versionnum < 5000 && $SectionsToLoad{"general"} && $FirstTime{$year.$month} && $LastTime{$year.$month}) ) { if ($Debug) { debug(" End of GENERAL section"); } # Show migrate warning for backward compatibility if ($versionnum < 5000 && ! $MigrateStats && ! $BadFormatWarning{$year.$month}) { if ($FrameName ne 'mainleft') { $BadFormatWarning{$year.$month}=1; my $message="Warning: Data file '$filetoread' has an old history file format (version $versionnum). You should upgrade it...\nFrom command line: $PROG.$Extension -migrate=\"$filetoread\""; if ($ENV{'GATEWAY_INTERFACE'} && $AllowToUpdateStatsFromBrowser) { $message.="\nFrom your browser with URL: http://".$ENV{"SERVER_NAME"}.$ENV{"SCRIPT_NAME"}."?migrate=$filetoread"; } warning("$message"); } } if (! ($versionnum < 5000) && $MigrateStats && ! $BadFormatWarning{$year.$month}) { $BadFormatWarning{$year.$month}=1; warning("Warning: You are migrating a file that is already a recent version (migrate not required for files version $versionnum).","","",1); } # If migrate and version < 4.x we need to include BEGIN_UNKNOWNIP into BEGIN_VISITOR for backward compatibility if ($MigrateStats && $versionnum < 4000) { if ($Debug) { debug("File is version < 4000. We add UNKNOWNIP in sections to load",1); } $SectionsToLoad{'unknownip'}=99; } delete $SectionsToLoad{'general'}; if ($SectionsToSave{'general'}) { Save_History('general',$year,$month,$lastlinenb,$lastlineoffset,$lastlinechecksum); delete $SectionsToSave{'general'}; } # Test for backward compatibility if ($versionnum < 5000 && ! $withupdate) { # We must find another way to init MonthUnique MonthHostsKnown and MonthHostsUnknown if ($Debug) { debug(" We ask to count MonthUnique, MonthHostsKnown and MonthHostsUnknown in visitor section because they are not stored in general section for this data file (version $versionnum)."); } $readvisitorforbackward=($SectionsToLoad{"visitor"}?1:2); $SectionsToLoad{"visitor"}=4; } else { if (! scalar %SectionsToLoad) { if ($Debug) { debug(" Stop reading history file. Got all we need."); } last; } } if ($versionnum >= 5000) { next; } # We can forget 'END_GENERAL' line and read next one } # BEGIN_MISC if ($field[0] eq 'BEGIN_MISC') { if ($Debug) { debug(" Begin of MISC section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'misc'}) { $countloaded++; if ($field[1]) { $_misc_p{$field[0]}+=int($field[1]); } if ($field[2]) { $_misc_h{$field[0]}+=int($field[2]); } if ($field[3]) { $_misc_k{$field[0]}+=int($field[3]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_MISC' || $field[0] eq "${xmleb}END_MISC" || ! $_); if ($field[0] ne 'END_MISC' && $field[0] ne "${xmleb}END_MISC") { error("History file \"$filetoread\" is corrupted (End of section MISC not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of MISC section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'misc'}; if ($SectionsToSave{'misc'}) { Save_History('misc',$year,$month); delete $SectionsToSave{'misc'}; if ($withpurge) { %_misc_p=(); %_misc_h=(); %_misc_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_CLUSTER if ($field[0] eq 'BEGIN_CLUSTER') { if ($Debug) { debug(" Begin of CLUSTER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'cluster'}) { $countloaded++; if ($field[1]) { $_cluster_p{$field[0]}+=int($field[1]); } if ($field[2]) { $_cluster_h{$field[0]}+=int($field[2]); } if ($field[3]) { $_cluster_k{$field[0]}+=int($field[3]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_CLUSTER' || $field[0] eq "${xmleb}END_CLUSTER" || ! $_); if ($field[0] ne 'END_CLUSTER' && $field[0] ne "${xmleb}END_CLUSTER") { error("History file \"$filetoread\" is corrupted (End of section CLUSTER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of CLUSTER section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'cluster'}; if ($SectionsToSave{'cluster'}) { Save_History('cluster',$year,$month); delete $SectionsToSave{'cluster'}; if ($withpurge) { %_cluster_p=(); %_cluster_h=(); %_cluster_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_TIME if ($field[0] eq 'BEGIN_TIME') { my $monthpages=0;my $monthhits=0;my $monthbytes=0; my $monthnotviewedpages=0;my $monthnotviewedhits=0;my $monthnotviewedbytes=0; if ($Debug) { debug(" Begin of TIME section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0] ne '') { # Test on ne '' because field[0] is '0' for hour 0) $count++; if ($SectionsToLoad{'time'}) { if ($withupdate || $MonthRequired eq 'all' || $MonthRequired eq "$month") { # Still required $countloaded++; if ($field[1]) { $_time_p[$field[0]]+=int($field[1]); } if ($field[2]) { $_time_h[$field[0]]+=int($field[2]); } if ($field[3]) { $_time_k[$field[0]]+=int($field[3]); } if ($field[4]) { $_time_nv_p[$field[0]]+=int($field[4]); } if ($field[5]) { $_time_nv_h[$field[0]]+=int($field[5]); } if ($field[6]) { $_time_nv_k[$field[0]]+=int($field[6]); } } $monthpages+=int($field[1]); $monthhits+=int($field[2]); $monthbytes+=int($field[3]); $monthnotviewedpages+=int($field[4]||0); $monthnotviewedhits+=int($field[5]||0); $monthnotviewedbytes+=int($field[6]||0); } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_TIME' || $field[0] eq "${xmleb}END_TIME" || ! $_); if ($field[0] ne 'END_TIME' && $field[0] ne "${xmleb}END_TIME") { error("History file \"$filetoread\" is corrupted (End of section TIME not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of TIME section ($count entries, $countloaded loaded)"); } $MonthPages{$year.$month}+=$monthpages; $MonthHits{$year.$month}+=$monthhits; $MonthBytes{$year.$month}+=$monthbytes; $MonthNotViewedPages{$year.$month}+=$monthnotviewedpages; $MonthNotViewedHits{$year.$month}+=$monthnotviewedhits; $MonthNotViewedBytes{$year.$month}+=$monthnotviewedbytes; delete $SectionsToLoad{'time'}; if ($SectionsToSave{'time'}) { Save_History('time',$year,$month); delete $SectionsToSave{'time'}; if ($withpurge) { @_time_p=(); @_time_h=(); @_time_k=(); @_time_nv_p=(); @_time_nv_h=(); @_time_nv_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_ORIGIN if ($field[0] eq 'BEGIN_ORIGIN') { if ($Debug) { debug(" Begin of ORIGIN section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'origin'}) { if ($field[0] eq 'From0') { $_from_p[0]+=$field[1]; $_from_h[0]+=$field[2]; } elsif ($field[0] eq 'From1') { $_from_p[1]+=$field[1]; $_from_h[1]+=$field[2]; } elsif ($field[0] eq 'From2') { $_from_p[2]+=$field[1]; $_from_h[2]+=$field[2]; } elsif ($field[0] eq 'From3') { $_from_p[3]+=$field[1]; $_from_h[3]+=$field[2]; } elsif ($field[0] eq 'From4') { $_from_p[4]+=$field[1]; $_from_h[4]+=$field[2]; } elsif ($field[0] eq 'From5') { $_from_p[5]+=$field[1]; $_from_h[5]+=$field[2]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_ORIGIN' || $field[0] eq "${xmleb}END_ORIGIN" || ! $_); if ($field[0] ne 'END_ORIGIN' && $field[0] ne "${xmleb}END_ORIGIN") { error("History file \"$filetoread\" is corrupted (End of section ORIGIN not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of ORIGIN section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'origin'}; if ($SectionsToSave{'origin'}) { Save_History('origin',$year,$month); delete $SectionsToSave{'origin'}; if ($withpurge) { @_from_p=(); @_from_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_DAY if ($field[0] eq 'BEGIN_DAY') { if ($Debug) { debug(" Begin of DAY section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'day'}) { $countloaded++; if ($field[1]) { $DayPages{$field[0]}+=int($field[1]); } $DayHits{$field[0]}+=int($field[2]); # DayHits always load (should be >0 and if not it's a day YYYYMM00 resulting of an old file migration) if ($field[3]) { $DayBytes{$field[0]}+=int($field[3]); } if ($field[4]) { $DayVisits{$field[0]}+=int($field[4]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_DAY' || $field[0] eq "${xmleb}END_DAY" || ! $_); if ($field[0] ne 'END_DAY' && $field[0] ne "${xmleb}END_DAY") { error("History file \"$filetoread\" is corrupted (End of section DAY not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of DAY section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'day'}; # WE DO NOT SAVE SECTION NOW BECAUSE VALUES CAN BE CHANGED AFTER READING VISITOR #if ($SectionsToSave{'day'}) { # Must be made after read of visitor # Save_History('day',$year,$month); delete $SectionsToSave{'day'}; # if ($withpurge) { %DayPages=(); %DayHits=(); %DayBytes=(); %DayVisits=(); } #} if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_VISITOR if ($field[0] eq 'BEGIN_VISITOR') { if ($Debug) { debug(" Begin of VISITOR section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; # For backward compatibility if ($readvisitorforbackward) { if ($field[1]) { $MonthUnique{$year.$month}++; } if ($MonthRequired ne 'all') { if ($field[0] !~ /^\d+\.\d+\.\d+\.\d+$/ && $field[0] !~ /^[0-9A-F]*:/i) { $MonthHostsKnown{$year.$month}++; } else { $MonthHostsUnknown{$year.$month}++; } } } # Process data saved in 'wait' arrays if ($withupdate && $_waithost_e{$field[0]}){ my $timehostl=int($field[4]||0); my $timehosts=int($field[5]||0); my $newtimehosts=($_waithost_s{$field[0]}?$_waithost_s{$field[0]}:$_host_s{$field[0]}); my $newtimehostl=($_waithost_l{$field[0]}?$_waithost_l{$field[0]}:$_host_l{$field[0]}); if ($newtimehosts > $timehostl + $VISITTIMEOUT ) { if ($Debug) { debug(" Visit for $field[0] in 'wait' arrays is a new visit different than last in history",4); } if ($field[6]) { $_url_x{$field[6]}++; } $_url_e{$_waithost_e{$field[0]}}++; $newtimehosts =~ /^(\d\d\d\d\d\d\d\d)/; $DayVisits{$1}++; if ($timehosts && $timehostl) { $_session{GetSessionRange($timehosts,$timehostl)}++; } if ($_waithost_s{$field[0]}) { # First session found in log was followed by another one so it's finished $_session{GetSessionRange($newtimehosts,$newtimehostl)}++; } # Here $_host_l $_host_s and $_host_u are correctly defined } else { if ($Debug) { debug(" Visit for $field[0] in 'wait' arrays is following of last visit in history",4); } if ($_waithost_s{$field[0]}) { # First session found in log was followed by another one so it's finished $_session{GetSessionRange(MinimumButNoZero($timehosts,$newtimehosts),$timehostl>$newtimehostl?$timehostl:$newtimehostl)}++; # Here $_host_l $_host_s and $_host_u are correctly defined } else { # We correct $_host_l $_host_s and $_host_u if ($timehostl > $newtimehostl) { $_host_l{$field[0]}=$timehostl; $_host_u{$field[0]}=$field[6]; } if ($timehosts < $newtimehosts) { $_host_s{$field[0]}=$timehosts; } } } delete $_waithost_e{$field[0]}; delete $_waithost_l{$field[0]}; delete $_waithost_s{$field[0]}; delete $_waithost_u{$field[0]}; } # Load records if ($readvisitorforbackward!=2 && $SectionsToLoad{'visitor'}) { # if readvisitorforbackward==2 we do not load my $loadrecord=0; if ($withupdate) { $loadrecord=1; } else { if ($HTMLOutput{'allhosts'} || $HTMLOutput{'lasthosts'}) { if ((!$FilterIn{'host'} || $field[0] =~ /$FilterIn{'host'}/i) && (!$FilterEx{'host'} || $field[0] !~ /$FilterEx{'host'}/i)) { $loadrecord=1; } } elsif ($MonthRequired eq 'all' || $field[2] >= $MinHit{'Host'}) { if ($HTMLOutput{'unknownip'} && ($field[0] =~ /^\d+\.\d+\.\d+\.\d+$/ || $field[0] =~ /^[0-9A-F]*:/i)) { $loadrecord=1; } elsif ($HTMLOutput{'main'} && ($MonthRequired eq 'all' || $countloaded < $MaxNbOf{'HostsShown'})) { $loadrecord=1; } } } if ($loadrecord) { if ($field[1]) { $_host_p{$field[0]}+=$field[1]; } if ($field[2]) { $_host_h{$field[0]}+=$field[2]; } if ($field[3]) { $_host_k{$field[0]}+=$field[3]; } if ($field[4] && ! $_host_l{$field[0]}) { # We save last connexion params if not previously defined $_host_l{$field[0]}=int($field[4]); if ($withupdate) { # field[5] field[6] are used only for update if ($field[5] && ! $_host_s{$field[0]}) { $_host_s{$field[0]}=int($field[5]); } if ($field[6] && ! $_host_u{$field[0]}) { $_host_u{$field[0]}=$field[6]; } } } $countloaded++; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_VISITOR' || $field[0] eq "${xmleb}END_VISITOR" || ! $_); if ($field[0] ne 'END_VISITOR' && $field[0] ne "${xmleb}END_VISITOR") { error("History file \"$filetoread\" is corrupted (End of section VISITOR not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of VISITOR section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'visitor'}; # WE DO NOT SAVE SECTION NOW TO BE SURE TO HAVE THIS LARGE SECTION NOT AT THE BEGINNING OF FILE #if ($SectionsToSave{'visitor'}) { # Save_History('visitor',$year,$month); delete $SectionsToSave{'visitor'}; # if ($withpurge) { %_host_p=(); %_host_h=(); %_host_k=(); %_host_l=(); %_host_s=(); %_host_u=(); } #} if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_UNKNOWNIP for backward compatibility if ($field[0] eq 'BEGIN_UNKNOWNIP') { my %iptomigrate=(); if ($Debug) { debug(" Begin of UNKNOWNIP section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'unknownip'}) { $iptomigrate{$field[0]}=$field[1]||0; $countloaded++; } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_UNKNOWNIP' || $field[0] eq "${xmleb}END_UNKNOWNIP" || ! $_); if ($field[0] ne 'END_UNKNOWNIP' && $field[0] ne "${xmleb}END_UNKNOWNIP") { error("History file \"$filetoread\" is corrupted (End of section UNKOWNIP not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of UNKOWNIP section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'visitor'}; # THIS SECTION IS NEVER SAVED. ONLY READ FOR MIGRATE AND CONVERTED INTO VISITOR SECTION foreach (keys %iptomigrate) { $_host_p{$_}+=int($_host_p{'Unknown'}/$countloaded); $_host_h{$_}+=int($_host_h{'Unknown'}/$countloaded); $_host_k{$_}+=int($_host_k{'Unknown'}/$countloaded); if ($iptomigrate{$_} > 0) { $_host_l{$_}=$iptomigrate{$_} }; } delete $_host_p{'Unknown'}; delete $_host_h{'Unknown'}; delete $_host_k{'Unknown'}; delete $_host_l{'Unknown'}; if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_LOGIN if ($field[0] eq 'BEGIN_LOGIN') { if ($Debug) { debug(" Begin of LOGIN section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'login'}) { $countloaded++; if ($field[1]) { $_login_p{$field[0]}+=$field[1]; } if ($field[2]) { $_login_h{$field[0]}+=$field[2]; } if ($field[3]) { $_login_k{$field[0]}+=$field[3]; } if (! $_login_l{$field[0]} && $field[4]) { $_login_l{$field[0]}=int($field[4]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_LOGIN' || $field[0] eq "${xmleb}END_LOGIN" || ! $_); if ($field[0] ne 'END_LOGIN' && $field[0] ne "${xmleb}END_LOGIN") { error("History file \"$filetoread\" is corrupted (End of section LOGIN not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of LOGIN section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'login'}; if ($SectionsToSave{'login'}) { Save_History('login',$year,$month); delete $SectionsToSave{'login'}; if ($withpurge) { %_login_p=(); %_login_h=(); %_login_k=(); %_login_l=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_DOMAIN if ($field[0] eq 'BEGIN_DOMAIN') { if ($Debug) { debug(" Begin of DOMAIN section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'domain'}) { $countloaded++; if ($field[1]) { $_domener_p{$field[0]}+=$field[1]; } if ($field[2]) { $_domener_h{$field[0]}+=$field[2]; } if ($field[3]) { $_domener_k{$field[0]}+=$field[3]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_DOMAIN' || $field[0] eq "${xmleb}END_DOMAIN" || ! $_); if ($field[0] ne 'END_DOMAIN' && $field[0] ne "${xmleb}END_DOMAIN") { error("History file \"$filetoread\" is corrupted (End of section DOMAIN not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of DOMAIN section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'domain'}; if ($SectionsToSave{'domain'}) { Save_History('domain',$year,$month); delete $SectionsToSave{'domain'}; if ($withpurge) { %_domener_p=(); %_domener_h=(); %_domener_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_SESSION if ($field[0] eq 'BEGIN_SESSION') { if ($Debug) { debug(" Begin of SESSION section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'session'}) { $countloaded++; if ($field[1]) { $_session{$field[0]}+=$field[1]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_SESSION' || $field[0] eq "${xmleb}END_SESSION" || ! $_); if ($field[0] ne 'END_SESSION' && $field[0] ne "${xmleb}END_SESSION") { error("History file \"$filetoread\" is corrupted (End of section SESSION not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of SESSION section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'session'}; # WE DO NOT SAVE SECTION NOW BECAUSE VALUES CAN BE CHANGED AFTER READING VISITOR #if ($SectionsToSave{'session'}) { # Save_History('session',$year,$month); delete $SectionsToSave{'session'}; } # if ($withpurge) { %_session=(); } #} if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_OS if ($field[0] eq 'BEGIN_OS') { if ($Debug) { debug(" Begin of OS section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'os'}) { $countloaded++; if ($field[1]) { $_os_h{$field[0]}+=$field[1]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_OS' || $field[0] eq "${xmleb}END_OS" || ! $_); if ($field[0] ne 'END_OS' && $field[0] ne "${xmleb}END_OS") { error("History file \"$filetoread\" is corrupted (End of section OS not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of OS section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'os'}; if ($SectionsToSave{'os'}) { Save_History('os',$year,$month); delete $SectionsToSave{'os'}; if ($withpurge) { %_os_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_BROWSER if ($field[0] eq 'BEGIN_BROWSER') { if ($Debug) { debug(" Begin of BROWSER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'browser'}) { $countloaded++; if ($field[1]) { $_browser_h{$field[0]}+=$field[1]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_BROWSER' || $field[0] eq "${xmleb}END_BROWSER" || ! $_); if ($field[0] ne 'END_BROWSER' && $field[0] ne "${xmleb}END_BROWSER") { error("History file \"$filetoread\" is corrupted (End of section BROWSER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of BROWSER section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'browser'}; if ($SectionsToSave{'browser'}) { Save_History('browser',$year,$month); delete $SectionsToSave{'browser'}; if ($withpurge) { %_browser_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_UNKNOWNREFERER if ($field[0] eq 'BEGIN_UNKNOWNREFERER') { if ($Debug) { debug(" Begin of UNKNOWNREFERER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'unknownreferer'}) { $countloaded++; if (! $_unknownreferer_l{$field[0]}) { $_unknownreferer_l{$field[0]}=int($field[1]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($readxml?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_UNKNOWNREFERER' || $field[0] eq "${xmleb}END_UNKNOWNREFERER" || ! $_); if ($field[0] ne 'END_UNKNOWNREFERER' && $field[0] ne "${xmleb}END_UNKNOWNREFERER") { error("History file \"$filetoread\" is corrupted (End of section UNKNOWNREFERER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for