Ticket #467: shp2pgsql_iconv_error_policy.patch3

Index: loader/shp2pgsql-core.c
===================================================================
--- loader/shp2pgsql-core.c     (revision 5557)
+++ loader/shp2pgsql-core.c     (working copy)
@@ -40,7 +40,7 @@
  * Internal functions
  */
 
-char *utf8(const char *fromcode, char *inputbuf);
+char *utf8(const SHPLOADERCONFIG *config, char *inputbuf);
 void vasbappend(stringbuffer_t *sb, char *fmt, ... );
 char *escape_copy_string(char *str);
 char *escape_insert_string(char *str);
@@ -77,7 +77,7 @@
 
 /* Return allocated string containing UTF8 string converted from encoding fromcode */
 char *
-utf8(const char *fromcode, char *inputbuf)
+utf8(const SHPLOADERCONFIG *config, char *inputbuf)
 {
        iconv_t cd;
        char *inbufptr = inputbuf;
@@ -88,7 +88,7 @@
 
        inbytesleft = strlen(inputbuf);
 
-       cd = iconv_open("UTF-8", fromcode);
+       cd = iconv_open(config->target_encoding, config->encoding);
        if ( cd == ((iconv_t)(-1)) )
                return NULL;
 
@@ -421,7 +421,7 @@
        {
                lwcollection = lwcollection_construct(MULTILINETYPE, state->config->sr_id, NULL, obj->nParts, lwmultilinestrings);
                serialized_lwgeom = lwgeom_serialize(lwcollection_as_lwgeom(lwcollection));
-               
+
        }
        else
        {
@@ -871,6 +871,7 @@
        config->createindex = 0;
        config->readshape = 1;
        config->encoding = strdup(ENCODING_DEFAULT);
+       config->target_encoding = DEFAULT_ICONV_POLICY;
        config->null_policy = POLICY_NULL_INSERT;
        config->sr_id = -1;
        config->hwgeom = 0;
@@ -1163,10 +1164,10 @@
                if (state->config->encoding)
                {
                        /* If we are converting from another encoding to UTF8, convert the field name to UTF8 */
-                       utf8str = utf8(state->config->encoding, name);
+                       utf8str = utf8(state->config, name);
                        if (!utf8str)
                        {
-                               snprintf(state->message, SHPLOADERMSGLEN, "Unable to convert field name \"%s\" from %s encoding to UTF-8: iconv reports \"%s\"", name, state->config->encoding, strerror(errno));
+                               snprintf(state->message, SHPLOADERMSGLEN, "Unable to convert field name \"%s\" from %s encoding to %s: iconv reports \"%s\"", name, state->config->encoding, state->config->target_encoding, strerror(errno));
                                return SHPLOADERERR;
                        }
 
@@ -1590,10 +1591,10 @@
                        if (state->config->encoding)
                        {
                                /* If we are converting from another encoding to UTF8, convert the field value to UTF8 */
-                               utf8str = utf8(state->config->encoding, val);
+                               utf8str = utf8(state->config, val);
                                if (!utf8str)
                                {
-                                       snprintf(state->message, SHPLOADERMSGLEN, "Unable to convert field value \"%s\" from %s encoding to UTF-8: iconv reports \"%s\"", val, state->config->encoding, strerror(errno));
+                                       snprintf(state->message, SHPLOADERMSGLEN, "Unable to convert field value \"%s\" from %s encoding to %s: iconv reports \"%s\"", val, state->config->encoding, state->config->target_encoding, strerror(errno));
                                        return SHPLOADERERR;
                                }
 
Index: loader/shp2pgsql-core.h
===================================================================
--- loader/shp2pgsql-core.h     (revision 5557)
+++ loader/shp2pgsql-core.h     (working copy)
@@ -40,6 +40,15 @@
 
 
 /*
+ * Error policies for iconv
+ */
+
+#define ICONV_POLICY_ERROR      "UTF-8"
+#define ICONV_POLICY_IGNORE     "UTF-8//IGNORE"
+
+#define DEFAULT_ICONV_POLICY    ICONV_POLICY_ERROR
+
+/*
  * Error message handling
  */
 
@@ -71,7 +80,7 @@
 #define ENCODING_DEFAULT "WINDOWS-1252"
 
 /*
- * Structure to hold the loader configuration options 
+ * Structure to hold the loader configuration options
  */
 
 typedef struct shp_loader_config
@@ -86,7 +95,7 @@
        char *schema;
 
        /* geometry column name to use */
-       char *geom; 
+       char *geom;
 
        /* the shape file (without the .shp extension) */
        char *shp_file;
@@ -96,7 +105,7 @@
 
        /* 0 = MULTIPOLYGON/MULTILINESTRING, 1 = force to POLYGON/LINESTRING */
        int simple_geometries;
-       
+
        /* 0 = geometry, 1 = geography */
        int geography;
 
@@ -115,6 +124,9 @@
        /* iconv encoding name */
        char *encoding;
 
+       /* always UTF-8 but with different iconv error policies */
+       const char *target_encoding;
+
        /* how to handle nulls */
        int null_policy;
 
@@ -128,7 +140,7 @@
 
 
 /*
- * Structure to holder the current loader state 
+ * Structure to holder the current loader state
  */
 
 typedef struct shp_loader_state
@@ -138,7 +150,7 @@
 
        /* Shapefile handle */
        SHPHandle hSHPHandle;
-       
+
        /* Shapefile type */
        int shpfiletype;
 
@@ -182,6 +194,8 @@
        /* Last (error) message */
        char message[SHPLOADERMSGLEN];
 
+       /* iconv conversion descriptor */
+       iconv_t conv_desc;
 } SHPLOADERSTATE;
 
 
Index: loader/shp2pgsql-cli.c
===================================================================
--- loader/shp2pgsql-cli.c      (revision 5557)
+++ loader/shp2pgsql-cli.c      (working copy)
@@ -41,6 +41,7 @@
        printf("  -S  Generate simple geometries instead of MULTI geometries.\n");
        printf("  -W <encoding> Specify the character encoding of Shape's\n");
        printf("     attribute column. (default : \""ENCODING_DEFAULT"\")\n");
+       printf("  -C  Ignore character encoding errors in dbf file\n");
        printf("  -N <policy> NULL geometries handling policy (insert*,skip,abort)\n");
        printf("  -n  Only import DBF file.\n");
        printf("  -?  Display this help screen.\n");
@@ -68,7 +69,7 @@
        config = malloc(sizeof(SHPLOADERCONFIG));
        set_config_defaults(config);
 
-       while ((c = pgis_getopt(argc, argv, "kcdapGDs:Sg:iW:wIN:n")) != EOF)
+       while ((c = pgis_getopt(argc, argv, "kcdapGDs:Sg:iW:wIN:C")) != EOF)
        {
                switch (c)
                {
@@ -132,6 +133,9 @@
                        config->encoding = pgis_optarg;
                        break;
 
+               case 'C':
+                       config->target_encoding = ICONV_POLICY_IGNORE;
+                       break;
                case 'N':
                        switch (pgis_optarg[0])
                        {