Logs are logged into the rollingfile and not console after rollingfile appender got initalized - log4j2

Below are the programmatic configurations of consoleAppender and LogAppenders which I have converted for log4j upgrade from 1.. to 2.17.1 version.
//Programmatic configuration of ConsoleAppender log4j2
String pattern = "%d{DATE} [%p|%C{2}] %m%n";
ConfigurationBuilder<BuiltConfiguration> builder =
ConfigurationBuilderFactory.newConfigurationBuilder();
builder.setConfigurationName("DefaultLogger");
// Create pattern layout
LayoutComponentBuilder layoutBuilder = builder.newLayout("PatternLayout")
.addAttribute("pattern", pattern);
AppenderComponentBuilder appenderBuilder = builder.newAppender("Console", "CONSOLE")
.addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT);
appenderBuilder.add(layoutBuilder);
builder.add(appenderBuilder);
RootLoggerComponentBuilder rootLogger
= builder.newRootLogger(Level.DEBUG);
rootLogger.add(builder.newAppenderRef("Console"));
builder.add(rootLogger);
Configurator.initialize(builder.build());
Configurator.reconfigure(builder.build());
//RollingFileAppender Programmatic configuration log4j2:
CdasLogger() {
// CreateLogger takes a path for each logger from config file
loadLog = createLogger("load.log");
updateLog = createLogger("update.log");
userLog = createLogger("user.log");
}
private Logger createLogger(String logType) {
String pattern = "%d %M - %m%n";
String consolePattern = "%d{DATE} [%p|%C{2}] %m%n";
String fileLogName = "/app/app.log";
String filePattern = "/app/app.log-%d{MM-dd-yy}.log.gz";
System.out.println("logtype is::" + logType);
String path = ConfigReader.getStringValue(logType);
System.out.println(path);
String daily = "0 0 12 1/1 * ? *";
// Initializing the logger context
LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
ConfigurationBuilder<BuiltConfiguration> builder =
ConfigurationBuilderFactory.newConfigurationBuilder();
builder.setConfigurationName("rollingFileLogger");
//specifying the pattern layout
LayoutComponentBuilder layoutBuilder = builder.newLayout("PatternLayout")
.addAttribute("pattern", pattern);
//specifying the policy for rolling file
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
. addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "10MB"));
//create a rollingfile appender
AppenderComponentBuilder appenderBuilder = builder.newAppender("rollingFile", "RollingFile")
.addAttribute("fileName", path)
.addAttribute("filePattern", path+"-%d{MM-dd-yy-HH-mm-ss}.log.")
.add(layoutBuilder)
.addComponent(triggeringPolicy);
builder.add(appenderBuilder);
RootLoggerComponentBuilder rootLogger = builder.newRootLogger(Level.TRACE);
rootLogger.add(builder.newAppenderRef("rollingFile"));
builder.add(rootLogger);
ctx = Configurator.initialize(builder.build());
Configurator.reconfigure(builder.build());
return ctx.getLogger(logType); // return the logger to the caller
}
Console Appender gets initialized first and the logs are written. After Initializing the rollingfileappender. All the logs are getting written to rollingfile Appender. There are no logs to consoleAppender once the rollingfile appender is initalized.
EDIT 1:
As per Piotr comments, I made the below change to have same configuration builder for all the appenders.
private void configureLogger() {
ConfigurationBuilder<BuiltConfiguration> builder = ConfigurationBuilderFactory.newConfigurationBuilder();
LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
// create a console appender
AppenderComponentBuilder console
= builder.newAppender("stdout", "Console").addAttribute("target",
ConsoleAppender.Target.SYSTEM_OUT);
console.add(builder.newLayout("PatternLayout").addAttribute("pattern",
"%d{DATE} [%p|%C{2}] %m%n"));
RootLoggerComponentBuilder rootLogger
= builder.newRootLogger(Level.DEBUG);
rootLogger.add(builder.newAppenderRef("stdout"));
builder.add(console);
// create a rolling file appender
String pattern = "%d %M - %m%n";
//specifying the pattern layout
LayoutComponentBuilder layoutBuilder = builder.newLayout("PatternLayout")
.addAttribute("pattern", pattern);
//specifying the policy for rolling file
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "10MB"));
String[] logTypes = {"load.log", "update.log", "user.log"};
for (String logType : logTypes) {
System.out.println("logtype is::" + logType);
String path = ConfigReader.getStringValue(logType);
System.out.println(path);
AppenderComponentBuilder appenderBuilder = builder.newAppender(logType, "RollingFile")
.addAttribute("fileName", path == null ? "/app1/app.log" : path)
.addAttribute("filePattern", path == null ? "/app1/app.log" : path+"-%d{MM-dd-yy-HH-mm-ss}.log.")
.add(layoutBuilder)
.addComponent(triggeringPolicy);
builder.add(appenderBuilder);
rootLogger.add(builder.newAppenderRef(logType));
}
builder.add(rootLogger);
Configurator.reconfigure(builder.build());
}
The logs are not updating still.
calling configureLogger() method during app start up.
In another class I have the code below
public class CdasLogger {
private final Logger updateLog, loadLog, userLog;
CdasLogger() {
loadLog = createLogger("load.log");
updateLog = createLogger("update.log");
userLog = createLogger("user.log");
}
private Logger createLogger(String logType) {
LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
return ctx.getLogger(logType);
}
}

Related

DefaultRolloverStrategy Log4j2 not working (programmatically)

I need to create a rollingfile appender and set the amount of logfiles during runtime with log4j2. So I use the following code to achive that:
LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
Configuration config = ctx.getConfiguration();
DefaultRolloverStrategy strategy = DefaultRolloverStrategy.newBuilder()
.withMax("4")
.withMin("1")
.withFileIndex("max")
.withConfig(config)
.withCompressionLevelStr(Deflater.NO_COMPRESSION + "")
.build();
PatternLayout layout = PatternLayout.newBuilder().withConfiguration(config)
.withPattern("%d{yyyy-MM-dd HH:mm:ss.SSS} [%5p] %pid --- %-40.40logger{39} : %m%n%wEx")
.build();
RollingFileAppender appender = RollingFileAppender.newBuilder().setConfiguration(config)
.withName("TraceFileAppender")
.withLayout(layout)
.withFileName("log.log")
.withFilePattern("log.%d{yyyy-MM-ddHHmmSSS}.log")
.withPolicy(SizeBasedTriggeringPolicy.createPolicy("20KB")
.withStrategy(strategy)
.build();
appender.start();
config.addAppender(appender);
LoggerConfig loggerConfig = config.getRootLogger();
loggerConfig.setLevel(Level.toLevel("DEBUG"));
loggerConfig.addAppender(appender, null, null);
This is working fine except from the max amount of files... I`m getting more files than the 4 in my strategy.... what is wrong? Any help is welcome!
Thanks in advance,
Regards Peter
pfff took me a while but I got it working... Not a lot information about programmaticaly changing log4j2 appenders here so this is my solution:
LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
Configuration config = ctx.getConfiguration();
PathCondition[] pathConditions = new PathCondition[1];
pathConditions[0] = IfAccumulatedFileCount.createFileCountCondition(Integer.parseInt(expire));
DeleteAction action = DeleteAction.createDeleteAction("C:\\logs\\BuddyServer\\", true, 1, false, null, pathConditions, null, config);
Action[] actions = new Action[1];
actions[0] = action;
DefaultRolloverStrategy strategy = DefaultRolloverStrategy.newBuilder()
.withMax(expire)
.withCustomActions(actions)
.withMin("1")
.withFileIndex("max")
.withConfig(config)
.withCompressionLevelStr(Deflater.NO_COMPRESSION + "")
.build();
PatternLayout layout = PatternLayout.newBuilder().withConfiguration(config)
.withPattern("%d{yyyy-MM-dd HH:mm:ss.SSS} [%5p] %pid --- %-40.40logger{39} : %m%n%wEx")
.build();
RollingFileAppender appender = RollingFileAppender.newBuilder().setConfiguration(config)
.withName("TraceFileAppender")
.withLayout(layout)
.withFileName(file + ".log")
.withFilePattern(file + ".%d{yyyy-MM-ddHHmmSSS}.log")
.withPolicy(SizeBasedTriggeringPolicy.createPolicy(segment))
.withStrategy(strategy)
.build();
appender.start();
config.addAppender(appender);
LoggerConfig loggerConfig = config.getRootLogger();
loggerConfig.setLevel(Level.toLevel(buddyConfig.getOption("log", "verbose").toUpperCase()));
loggerConfig.addAppender(appender, null, null);

update/set maxcpc on one specific shopping product in an adgroup

I want to update my max cpc bid for a specific product in adwords.
Via the webui of adwords this is a trivial task, but I cant get it to work in code, this is what I have so far.
import com.google.api.ads.adwords.axis.factory.AdWordsServices;
import com.google.api.ads.adwords.axis.v201607.cm.*;
import com.google.api.ads.adwords.lib.client.AdWordsSession;
import com.google.api.ads.common.lib.auth.OfflineCredentials;
import com.google.api.client.auth.oauth2.Credential;
import java.rmi.RemoteException;
public class ChangeBidOnSpecificProduct {
public static void main(String[] args) throws Exception {
OfflineCredentials build = new OfflineCredentials.Builder()
.forApi(OfflineCredentials.Api.ADWORDS)
.fromFile()
.build();
Credential oAuth2Credential = build
.generateCredential();
// Construct an AdWordsSession.
AdWordsSession session = new AdWordsSession.Builder()
.fromFile()
.withOAuth2Credential(oAuth2Credential)
.build();
String accountId = "ACCOUNT_ID";
Long campaignId = Long.valueOf("CAMPAIGN_ID");
long adGroupId = Long.valueOf("ADGROUP_ID");
session.setClientCustomerId(accountId);
Money money = new Money(null, 40000L);
String productId = "9200000050670959";
changeBidViaApi(session, campaignId, adGroupId, productId, money);
}
private static void changeBidViaApi(AdWordsSession session, Long campaignId, long adGroupId, String productId, Money newValue) throws RemoteException {
ProductOfferId productOfferId = new ProductOfferId();
productOfferId.setValue(productId);
ProductScope productScope = new ProductScope();
productScope.setDimensions(new ProductDimension[] {productOfferId});
BiddableAdGroupCriterion biddableAdGroupCriterion = new BiddableAdGroupCriterion();
biddableAdGroupCriterion.setAdGroupId(adGroupId);
biddableAdGroupCriterion.setCriterion(productScope);
BiddingStrategyConfiguration biddingStrategyConfiguration = new BiddingStrategyConfiguration();
CpcBid bid = new CpcBid();
bid.setBid(newValue);
biddingStrategyConfiguration.setBids(new Bids[]{bid});
biddableAdGroupCriterion.setBiddingStrategyConfiguration(biddingStrategyConfiguration);
AdGroupCriterionOperation operation = new AdGroupCriterionOperation();
operation.setOperand(biddableAdGroupCriterion);
operation.setOperator(Operator.SET);
AdGroupCriterionOperation[] operations = new AdGroupCriterionOperation[]{operation};
AdWordsServices adWordsServices = new AdWordsServices();
AdGroupCriterionServiceInterface adGroupCriterionService =
adWordsServices.get(session, AdGroupCriterionServiceInterface.class);
AdGroupCriterionReturnValue result = adGroupCriterionService.mutate(operations);
}
}
Executing this results in an error: 'Unmarshalling Error: cvc-elt.4.2: Cannot resolve 'ns2:ProductScope' to a type definition for element 'ns2:criterion''.
This is the (anonimized) data that is send to google:
<?xml version="1.0" encoding="UTF-8"?>
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<soapenv:Header>
<ns1:RequestHeader soapenv:mustUnderstand="0" xmlns:ns1="https://adwords.google.com/api/adwords/cm/v201607">
<ns1:clientCustomerId>ACCOUNT_ID</ns1:clientCustomerId>
<ns1:developerToken>MY_DEV_TOKEN</ns1:developerToken>
<ns1:userAgent>MY_UA</ns1:userAgent>
<ns1:validateOnly>false</ns1:validateOnly>
<ns1:partialFailure>false</ns1:partialFailure>
</ns1:RequestHeader>
</soapenv:Header>
<soapenv:Body>
<mutate xmlns="https://adwords.google.com/api/adwords/cm/v201607">
<operations>
<operator>SET</operator>
<operand xsi:type="ns2:BiddableAdGroupCriterion" xmlns:ns2="https://adwords.google.com/api/adwords/cm/v201607">
<ns2:adGroupId>ADGROUP_ID</ns2:adGroupId>
<ns2:criterion xsi:type="ns2:ProductScope">
<ns2:dimensions xsi:type="ns2:ProductOfferId">
<ns2:value>9200000050670959</ns2:value>
</ns2:dimensions>
</ns2:criterion>
<ns2:biddingStrategyConfiguration>
<ns2:bids xsi:type="ns2:CpcBid">
<ns2:bid>
<ns2:microAmount>1</ns2:microAmount>
</ns2:bid>
</ns2:bids>
</ns2:biddingStrategyConfiguration>
</operand>
</operations>
</mutate>
</soapenv:Body>
</soapenv:Envelope>
Any suggestions as the reason for the error or what I'm doing wrong here?
I managed to get it working by accessing the adwords items as a graph, iterating it and changing it bid value when I find a match.
private static void changeBidViaApi(AdWordsSession session, long adGroupId, String productId, Money newValue) throws RemoteException {
AdWordsServices adWordsServices = new AdWordsServices();
ProductPartitionTree partitionTree =
ProductPartitionTree.createAdGroupTree(adWordsServices, session, adGroupId);
for (ProductPartitionNode node : partitionTree.getRoot().getChildren()) {
ProductPartitionNode productPartitionNode = node.asBiddableUnit();
try {
ProductOfferId dimension = (ProductOfferId) productPartitionNode.getDimension();
if (dimension != null) {
String productIdInShopping = dimension.getValue();
if (productId.equals(productIdInShopping)) {
Long newBid = newValue.getMicroAmount();
productPartitionNode.setBid(newBid);
}
}
} catch (Exception exception) {
exception.printStackTrace();
}
}
AdGroupCriterionServiceInterface adGroupCriterionService =
adWordsServices.get(session, AdGroupCriterionServiceInterface.class);
List<AdGroupCriterionOperation> mutateOperations = partitionTree.getMutateOperations();
if (mutateOperations.isEmpty()) {
System.out.println("Nothing to do.");
} else {
adGroupCriterionService.mutate(mutateOperations.toArray(new AdGroupCriterionOperation[0]));
}
}
This is the working source code. It changes bid (max cpc) of product partition group. It is written in php, but you can modify this code to java.
$adWordsServices = new AdWordsServices();
$session = $this->getSession();
$adGroupCriterionService = $adWordsServices->get($session, AdGroupCriterionService::class);
$operations = [];
$adGroupCriterion = new BiddableAdGroupCriterion();
$adGroupCriterion->setAdGroupId(22122723325); // id of my adgroup
$adGroupCriterion->setCriterion(new Criterion(302190832)); // id of partition group. you can get find this id in PRODUCT_PARTITION_REPORT in ID field (which full name is Criterion ID)
//
$bid = new CpcBid();
$money = new Money();
$money->setMicroAmount(((float)4)*1000000);
$bid->setBid($money);
$biddingStrategyConfiguration = new BiddingStrategyConfiguration();
$biddingStrategyConfiguration->setBids([$bid]);
$adGroupCriterion->setBiddingStrategyConfiguration($biddingStrategyConfiguration);
$operation = new AdGroupCriterionOperation();
$operation->setOperand($adGroupCriterion);
$operation->setOperator(Operator::SET);
$operations[] = $operation;
//
$adGroupCriterionService->mutate($operations);

Configuring log4net Programmatically for an AdoNetAppender

I have following code:
public class Log4NetSetup
{
public static void Setup()
{
#region log4net parameters
AdoNetAppenderParameter logDate = new AdoNetAppenderParameter()
{
ParameterName = "#log_date",
DbType = System.Data.DbType.DateTime,
Layout = new RawTimeStampLayout()
};
AdoNetAppenderParameter thread = new AdoNetAppenderParameter()
{
ParameterName = "#thread",
DbType = System.Data.DbType.String,
Size = 255,
Layout = (IRawLayout)new PatternLayout("%thread")
};
AdoNetAppenderParameter logLevel = new AdoNetAppenderParameter()
{
ParameterName = "#log_level",
DbType = System.Data.DbType.String,
Size = 50,
Layout = (IRawLayout)new PatternLayout("%level")
};
AdoNetAppenderParameter logger = new AdoNetAppenderParameter()
{
ParameterName = "#logger",
DbType = System.Data.DbType.String,
Size = 255,
Layout = (IRawLayout)new PatternLayout("%logger")
};
AdoNetAppenderParameter message = new AdoNetAppenderParameter()
{
ParameterName = "#message",
DbType = System.Data.DbType.String,
Size = 4000,
Layout = (IRawLayout)new PatternLayout("%message")
};
AdoNetAppenderParameter exception = new AdoNetAppenderParameter()
{
ParameterName = "#exception",
DbType = System.Data.DbType.String,
Size = 2000,
Layout = (IRawLayout)new ExceptionLayout()
};
#endregion
AdoNetAppender dotNet = new AdoNetAppender()
{
BufferSize = 1,
CommandType = System.Data.CommandType.Text,
ConnectionType = "REDACTED",
ConnectionString = "REDACTED",
CommandText = "REDACTED",
};
dotNet.AddParameter(logDate);
dotNet.AddParameter(thread);
dotNet.AddParameter(logLevel);
dotNet.AddParameter(logger);
dotNet.AddParameter(message);
dotNet.AddParameter(exception);
dotNet.ActivateOptions();
Hierarchy hierarchy = (Hierarchy)LogManager.GetRepository();
hierarchy.Root.AddAppender(dotNet);
hierarchy.Root.Level = Level.Debug;
hierarchy.Configured = true;
//log4net.Config.BasicConfigurator.Configure(dotNet);
log4net.Config.BasicConfigurator.Configure(hierarchy);
}
}
Then I run that in Global.Asax on startup.
This was working fine in a XML file, but when moving it to code it no longer wants to work.
I was moving this into code and followed many other ways to do this on SO.
None of them seem to work in my situation.
How to configure log4net programmatically from scratch (no config)
and
Can you configure log4net in code instead of using a config file?
Turns out there was an error, it was not causing a problem with the page though.
There was a invalid cast with the following code:
AdoNetAppenderParameter thread = new AdoNetAppenderParameter()
{
ParameterName = "#thread",
DbType = System.Data.DbType.String,
Size = 255,
Layout = (IRawLayout)new PatternLayout("%thread")
};
Specifically:
Layout = (IRawLayout)new PatternLayout("%thread")
It needed to convert the type:
RawLayoutConverter rlc = new RawLayoutConverter();
AdoNetAppenderParameter thread = new AdoNetAppenderParameter()
{
ParameterName = "#thread",
DbType = System.Data.DbType.String,
Size = 255,
Layout = (IRawLayout)rlc.ConvertFrom(new PatternLayout("%thread"))
};

Crawler4j With Grails App

I am making a crawler application in Groovy on Grails. I am using Crawler4j and following this tutorial.
I created a new grails project
Put the BasicCrawlController.groovy file in controllers->package
Did not create any view because I expected on doing run-app, my crawled data would appear in my crawlStorageFolder (please correct me if my understanding is flawed)
After that I just ran the application by doing run-app but I didn't see any crawling data anywhere.
Am I right in expecting some file to be created at the crawlStorageFolder location that I have given as C:/crawl/crawler4jStorage?
Do I need to create any view for this?
If I want to invoke this crawler controller from some other view on click of a submit button of a form, can I just write <g:form name="submitWebsite" url="[controller:'BasicCrawlController ']">?
I asked this because I do not have any method in this controller, so is it the right way to invoke this controller?
My code is as follows:
//All necessary imports
public class BasicCrawlController {
static main(args) throws Exception {
String crawlStorageFolder = "C:/crawl/crawler4jStorage";
int numberOfCrawlers = 1;
//int maxDepthOfCrawling = -1; default
CrawlConfig config = new CrawlConfig();
config.setCrawlStorageFolder(crawlStorageFolder);
config.setPolitenessDelay(1000);
config.setMaxPagesToFetch(100);
config.setResumableCrawling(false);
PageFetcher pageFetcher = new PageFetcher(config);
RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
controller.addSeed("http://en.wikipedia.org/wiki/Web_crawler")
controller.start(BasicCrawler.class, 1);
}
}
class BasicCrawler extends WebCrawler {
final static Pattern FILTERS = Pattern
.compile(".*(\\.(css|js|bmp|gif|jpe?g"+ "|png|tiff?|mid|mp2|mp3|mp4" +
"|wav|avi|mov|mpeg|ram|m4v|pdf" +"|rm|smil|wmv|swf|wma|zip|rar|gz))\$")
/**
* You should implement this function to specify whether the given url
* should be crawled or not (based on your crawling logic).
*/
#Override
boolean shouldVisit(WebURL url) {
String href = url.getURL().toLowerCase()
!FILTERS.matcher(href).matches() && href.startsWith("http://en.wikipedia.org/wiki/Web_crawler/")
}
/**
* This function is called when a page is fetched and ready to be processed
* by your program.
*/
#Override
void visit(Page page) {
int docid = page.getWebURL().getDocid()
String url = page.getWebURL().getURL()
String domain = page.getWebURL().getDomain()
String path = page.getWebURL().getPath()
String subDomain = page.getWebURL().getSubDomain()
String parentUrl = page.getWebURL().getParentUrl()
String anchor = page.getWebURL().getAnchor()
println("Docid: ${docid} ")
println("URL: ${url} ")
println("Domain: '${domain}'")
println("Sub-domain: ' ${subDomain}'")
println("Path: '${path}'")
println("Parent page:${parentUrl} ")
println("Anchor text: ${anchor} " )
if (page.getParseData() instanceof HtmlParseData) {
HtmlParseData htmlParseData = (HtmlParseData) page.getParseData()
String text = htmlParseData.getText()
String html = htmlParseData.getHtml()
List<WebURL> links = htmlParseData.getOutgoingUrls()
println("Text length: " + text.length())
println("Html length: " + html.length())
println("Number of outgoing links: " + links.size())
}
Header[] responseHeaders = page.getFetchResponseHeaders()
if (responseHeaders != null) {
println("Response headers:")
for (Header header : responseHeaders) {
println("\t ${header.getName()} : ${header.getValue()}")
}
}
println("=============")
}
}
I'll try to translate your code into a Grails standard.
Use this under grails-app/controller
class BasicCrawlController {
def index() {
String crawlStorageFolder = "C:/crawl/crawler4jStorage";
int numberOfCrawlers = 1;
//int maxDepthOfCrawling = -1; default
CrawlConfig crawlConfig = new CrawlConfig();
crawlConfig.setCrawlStorageFolder(crawlStorageFolder);
crawlConfig.setPolitenessDelay(1000);
crawlConfig.setMaxPagesToFetch(100);
crawlConfig.setResumableCrawling(false);
PageFetcher pageFetcher = new PageFetcher(crawlConfig);
RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
CrawlController controller = new CrawlController(crawlConfig, pageFetcher, robotstxtServer);
controller.addSeed("http://en.wikipedia.org/wiki/Web_crawler")
controller.start(BasicCrawler.class, 1);
render "done crawling"
}
}
Use this under src/groovy
class BasicCrawler extends WebCrawler {
final static Pattern FILTERS = Pattern
.compile(".*(\\.(css|js|bmp|gif|jpe?g"+ "|png|tiff?|mid|mp2|mp3|mp4" +
"|wav|avi|mov|mpeg|ram|m4v|pdf" +"|rm|smil|wmv|swf|wma|zip|rar|gz))\$")
/**
* You should implement this function to specify whether the given url
* should be crawled or not (based on your crawling logic).
*/
#Override
boolean shouldVisit(WebURL url) {
String href = url.getURL().toLowerCase()
!FILTERS.matcher(href).matches() && href.startsWith("http://en.wikipedia.org/wiki/Web_crawler/")
}
/**
* This function is called when a page is fetched and ready to be processed
* by your program.
*/
#Override
void visit(Page page) {
int docid = page.getWebURL().getDocid()
String url = page.getWebURL().getURL()
String domain = page.getWebURL().getDomain()
String path = page.getWebURL().getPath()
String subDomain = page.getWebURL().getSubDomain()
String parentUrl = page.getWebURL().getParentUrl()
String anchor = page.getWebURL().getAnchor()
println("Docid: ${docid} ")
println("URL: ${url} ")
println("Domain: '${domain}'")
println("Sub-domain: ' ${subDomain}'")
println("Path: '${path}'")
println("Parent page:${parentUrl} ")
println("Anchor text: ${anchor} " )
if (page.getParseData() instanceof HtmlParseData) {
HtmlParseData htmlParseData = (HtmlParseData) page.getParseData()
String text = htmlParseData.getText()
String html = htmlParseData.getHtml()
List<WebURL> links = htmlParseData.getOutgoingUrls()
println("Text length: " + text.length())
println("Html length: " + html.length())
println("Number of outgoing links: " + links.size())
}
Header[] responseHeaders = page.getFetchResponseHeaders()
if (responseHeaders != null) {
println("Response headers:")
for (Header header : responseHeaders) {
println("\t ${header.getName()} : ${header.getValue()}")
}
}
println("=============")
}
}

Where's located the declaration of messageSource in Grails?

Background
We have some legacy internationalization for field labels that are stored in the database, so I tried to make a "merged" messageSource. If the code exists in database, return, if not, use PluginAwareResourceBundleMessageSource to look in the i18n.
Problem
For some reason the cachedMergedPluginProperties is caching the wrong file for the locale. For example, if I search for en_US, I receive pt_BR messages (the key of the Map is en_US, but the properties are pt_BR).
I declared my messageSource as follows:
messageSource(DatabaseMessageSource) {
messageBundleMessageSource = { org.codehaus.groovy.grails.context.support.PluginAwareResourceBundleMessageSource m ->
basenames = "WEB-INF/grails-app/i18n/messages"
}
}
The inner bean is beacause of Grails won't let me have two beans of type MessageSource.
Am I declaring PluginAwareResourceBundleMessageSource different from the default of Grails? In which file of Grails I can see this bean declaration?
I found the declaration inside I18nGrailsPlugin, and it's a bit more detailed then mine:
String baseDir = "grails-app/i18n"
String version = GrailsUtil.getGrailsVersion()
String watchedResources = "file:./${baseDir}/**/*.properties".toString()
...
Set baseNames = []
def messageResources
if (application.warDeployed) {
messageResources = parentCtx?.getResources("**/WEB-INF/${baseDir}/**/*.properties")?.toList()
}
else {
messageResources = plugin.watchedResources
}
if (messageResources) {
for (resource in messageResources) {
// Extract the file path of the file's parent directory
// that comes after "grails-app/i18n".
String path
if (resource instanceof ContextResource) {
path = StringUtils.substringAfter(resource.pathWithinContext, baseDir)
}
else {
path = StringUtils.substringAfter(resource.path, baseDir)
}
// look for an underscore in the file name (not the full path)
String fileName = resource.filename
int firstUnderscore = fileName.indexOf('_')
if (firstUnderscore > 0) {
// grab everyting up to but not including
// the first underscore in the file name
int numberOfCharsToRemove = fileName.length() - firstUnderscore
int lastCharacterToRetain = -1 * (numberOfCharsToRemove + 1)
path = path[0..lastCharacterToRetain]
}
else {
// Lop off the extension - the "basenames" property in the
// message source cannot have entries with an extension.
path -= ".properties"
}
baseNames << "WEB-INF/" + baseDir + path
}
}
LOG.debug "Creating messageSource with basenames: $baseNames"
messageSource(PluginAwareResourceBundleMessageSource) {
basenames = baseNames.toArray()
fallbackToSystemLocale = false
pluginManager = manager
if (Environment.current.isReloadEnabled() || GrailsConfigUtils.isConfigTrue(application, GroovyPagesTemplateEngine.CONFIG_PROPERTY_GSP_ENABLE_RELOAD)) {
def cacheSecondsSetting = application?.flatConfig?.get('grails.i18n.cache.seconds')
if (cacheSecondsSetting != null) {
cacheSeconds = cacheSecondsSetting as Integer
} else {
cacheSeconds = 5
}
}
}
Since Grails don't let you have two beans of type MessageSource I had to copy this code and adapt to mine "merged" messageSource.

Resources