[Crash-utility] [PATCH 01/16] Add MIPS64 framework code support

Youling Tang tangyouling at loongson.cn
Wed Mar 10 09:17:33 UTC 2021


Hi Kazu,
Thank you for your reply and suggestion.

On 03/10/2021 09:13 AM, HAGIO KAZUHITO(萩尾 一仁) wrote:
> Hi Youling,
>
> (Please find my comment inline.)
>
> -----Original Message-----
>> Mainly added some environment configurations, macro definitions, specific
>> architecture structures and some function declarations supported by the
>> MIPS64 architecture.
>>
>> Signed-off-by: Huacai Chen <chenhuacai at loongson.cn>
>> Signed-off-by: Youling Tang <tangyouling at loongson.cn>
>> ---
>>   Makefile            |   7 ++-
>>   configure.c         |  33 ++++++++++--
>>   defs.h              | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++-
>>   diskdump.c          |   3 ++
>>   lkcd_vmdump_v2_v3.h |   3 +-
>>   mips64.c            |  45 +++++++++++++++++
>>   symbols.c           |  10 ++++
>>   7 files changed, 236 insertions(+), 8 deletions(-)
>>   create mode 100644 mips64.c
>>
>> diff --git a/Makefile b/Makefile
>> index f66eba7..eb1e018 100644
>> --- a/Makefile
>> +++ b/Makefile
>> @@ -63,7 +63,7 @@ CFILES=main.c tools.c global_data.c memory.c filesys.c help.c task.c \
>>   	kernel.c test.c gdb_interface.c configure.c net.c dev.c bpf.c \
>>   	printk.c \
>>   	alpha.c x86.c ppc.c ia64.c s390.c s390x.c s390dbf.c ppc64.c x86_64.c \
>> -	arm.c arm64.c mips.c sparc64.c \
>> +	arm.c arm64.c mips.c mips64.c sparc64.c \
>>   	extensions.c remote.c va_server.c va_server_v1.c symbols.c cmdline.c \
>>   	lkcd_common.c lkcd_v1.c lkcd_v2_v3.c lkcd_v5.c lkcd_v7.c lkcd_v8.c\
>>   	lkcd_fix_mem.c s390_dump.c lkcd_x86_trace.c \
>> @@ -83,7 +83,7 @@ OBJECT_FILES=main.o tools.o global_data.o memory.o filesys.o help.o task.o \
>>   	build_data.o kernel.o test.o gdb_interface.o net.o dev.o bpf.o \
>>   	printk.o \
>>   	alpha.o x86.o ppc.o ia64.o s390.o s390x.o s390dbf.o ppc64.o x86_64.o \
>> -	arm.o arm64.o mips.o sparc64.o \
>> +	arm.o arm64.o mips.o mips64.o sparc64.o \
>>   	extensions.o remote.o va_server.o va_server_v1.o symbols.o cmdline.o \
>>   	lkcd_common.o lkcd_v1.o lkcd_v2_v3.o lkcd_v5.o lkcd_v7.o lkcd_v8.o \
>>   	lkcd_fix_mem.o s390_dump.o netdump.o diskdump.o makedumpfile.o xendump.o \
>> @@ -434,6 +434,9 @@ arm64.o: ${GENERIC_HFILES} ${REDHAT_HFILES} arm64.c
>>   mips.o: ${GENERIC_HFILES} ${REDHAT_HFILES} mips.c
>>   	${CC} -c ${CRASH_CFLAGS} mips.c ${WARNING_OPTIONS} ${WARNING_ERROR}
>>
>> +mips64.o: ${GENERIC_HFILES} ${REDHAT_HFILES} mips64.c
>> +	${CC} -c ${CRASH_CFLAGS} mips64.c ${WARNING_OPTIONS} ${WARNING_ERROR}
>> +
>>   sparc64.o: ${GENERIC_HFILES} ${REDHAT_HFILES} sparc64.c
>>   	${CC} -c ${CRASH_CFLAGS} sparc64.c ${WARNING_OPTIONS} ${WARNING_ERROR}
>>
>> diff --git a/configure.c b/configure.c
>> index 7f6d19e..e2df6f0 100644
>> --- a/configure.c
>> +++ b/configure.c
>> @@ -104,7 +104,9 @@ void add_extra_lib(char *);
>>   #undef X86_64
>>   #undef ARM
>>   #undef ARM64
>> +#undef MIPS
>>   #undef SPARC64
>> +#undef MIPS64
>>
>>   #define UNKNOWN 0
>>   #define X86     1
>> @@ -119,6 +121,7 @@ void add_extra_lib(char *);
>>   #define ARM64   10
>>   #define MIPS    11
>>   #define SPARC64 12
>> +#define MIPS64  13
>>
>>   #define TARGET_X86    "TARGET=X86"
>>   #define TARGET_ALPHA  "TARGET=ALPHA"
>> @@ -131,6 +134,7 @@ void add_extra_lib(char *);
>>   #define TARGET_ARM    "TARGET=ARM"
>>   #define TARGET_ARM64  "TARGET=ARM64"
>>   #define TARGET_MIPS   "TARGET=MIPS"
>> +#define TARGET_MIPS64 "TARGET=MIPS64"
>>   #define TARGET_SPARC64 "TARGET=SPARC64"
>>
>>   #define TARGET_CFLAGS_X86    "TARGET_CFLAGS=-D_FILE_OFFSET_BITS=64"
>> @@ -152,6 +156,7 @@ void add_extra_lib(char *);
>>   #define TARGET_CFLAGS_MIPS            "TARGET_CFLAGS=-D_FILE_OFFSET_BITS=64"
>>   #define TARGET_CFLAGS_MIPS_ON_X86     "TARGET_CFLAGS=-D_FILE_OFFSET_BITS=64"
>>   #define TARGET_CFLAGS_MIPS_ON_X86_64  "TARGET_CFLAGS=-m32 -D_FILE_OFFSET_BITS=64"
>> +#define TARGET_CFLAGS_MIPS64          "TARGET_CFLAGS="
>>   #define TARGET_CFLAGS_SPARC64         "TARGET_CFLAGS="
>>
>>   #define GDB_TARGET_DEFAULT        "GDB_CONF_FLAGS="
>> @@ -380,7 +385,11 @@ get_current_configuration(struct supported_gdb_version *sp)
>>           target_data.target = ARM64;
>>   #endif
>>   #ifdef __mips__
>> -        target_data.target = MIPS;
>> +#ifndef __mips64
>> +	target_data.target = MIPS;
>> +#else
>> +	target_data.target = MIPS64;
>> +#endif
>>   #endif
>>   #ifdef __sparc_v9__
>>   	target_data.target = SPARC64;
>> @@ -474,6 +483,10 @@ get_current_configuration(struct supported_gdb_version *sp)
>>   				arch_mismatch(sp);
>>   		}
>>
>> +		if ((target_data.initial_gdb_target == MIPS64) &&
>> +		(target_data.target != MIPS64))
>> +			arch_mismatch(sp);
>> +
>>   		if ((target_data.initial_gdb_target == X86) &&
>>   		    (target_data.target != X86)) {
>>   			if (target_data.target == X86_64)
>> @@ -631,6 +644,9 @@ show_configuration(void)
>>   	case MIPS:
>>   		printf("TARGET: MIPS\n");
>>   		break;
>> +	case MIPS64:
>> +		printf("TARGET: MIPS64\n");
>> +		break;
>>   	case SPARC64:
>>   		printf("TARGET: SPARC64\n");
>>   		break;
>> @@ -742,7 +758,11 @@ build_configure(struct supported_gdb_version *sp)
>>   			gdb_conf_flags = GDB_TARGET_MIPS_ON_X86_64;
>>   		} else
>>                           target_CFLAGS = TARGET_CFLAGS_MIPS;
>> -                break;
>> +		break;
>> +	case MIPS64:
>> +		target = TARGET_MIPS64;
>> +		target_CFLAGS = TARGET_CFLAGS_MIPS64;
>> +		break;
>>   	case SPARC64:
>>   		target = TARGET_SPARC64;
>>   		target_CFLAGS = TARGET_CFLAGS_SPARC64;
>> @@ -1344,7 +1364,7 @@ make_spec_file(struct supported_gdb_version *sp)
>>   	printf("Vendor: Red Hat, Inc.\n");
>>   	printf("Packager: Dave Anderson <anderson at redhat.com>\n");
>>   	printf("ExclusiveOS: Linux\n");
>> -	printf("ExclusiveArch: %%{ix86} alpha ia64 ppc ppc64 ppc64pseries ppc64iseries x86_64 s390 s390x
>> arm aarch64 ppc64le mips mipsel sparc64\n");
>> +	printf("ExclusiveArch: %%{ix86} alpha ia64 ppc ppc64 ppc64pseries ppc64iseries x86_64 s390 s390x
>> arm aarch64 ppc64le mips mipsel mips64el sparc64\n");
>>   	printf("Buildroot: %%{_tmppath}/%%{name}-root\n");
>>   	printf("BuildRequires: ncurses-devel zlib-devel bison\n");
>>   	printf("Requires: binutils\n");
>> @@ -1571,6 +1591,8 @@ set_initial_target(struct supported_gdb_version *sp)
>>   		target_data.initial_gdb_target = ARM64;
>>   	else if (strncmp(buf, "ARM", strlen("ARM")) == 0)
>>   		target_data.initial_gdb_target = ARM;
>> +	else if (strncmp(buf, "MIPS64", strlen("MIPS64")) == 0)
>> +		target_data.initial_gdb_target = MIPS64;
>>   	else if (strncmp(buf, "MIPS", strlen("MIPS")) == 0)
>>   		target_data.initial_gdb_target = MIPS;
>>   	else if (strncmp(buf, "SPARC64", strlen("SPARC64")) == 0)
>> @@ -1593,6 +1615,7 @@ target_to_name(int target)
>>   	case ARM:    return("ARM");
>>   	case ARM64:  return("ARM64");
>>   	case MIPS:   return("MIPS");
>> +	case MIPS64: return("MIPS64");
>>   	case SPARC64: return("SPARC64");
>>   	}
>>
>> @@ -1652,6 +1675,10 @@ name_to_target(char *name)
>>                   return MIPS;
>>           else if (strncmp(name, "MIPS", strlen("MIPS")) == 0)
>>                   return MIPS;
>> +	else if (strncmp(name, "mips64", strlen("mips64")) == 0)
>> +		return MIPS64;
>> +	else if (strncmp(name, "MIPS64", strlen("MIPS64")) == 0)
>> +		return MIPS64;
>>   	else if (strncmp(name, "sparc64", strlen("sparc64")) == 0)
>>   		return SPARC64;
>>
>> diff --git a/defs.h b/defs.h
>> index ffbe73b..21cd904 100644
>> --- a/defs.h
>> +++ b/defs.h
>> @@ -71,7 +71,8 @@
>>
>>   #if !defined(X86) && !defined(X86_64) && !defined(ALPHA) && !defined(PPC) && \
>>       !defined(IA64) && !defined(PPC64) && !defined(S390) && !defined(S390X) && \
>> -    !defined(ARM) && !defined(ARM64) && !defined(MIPS) && !defined(SPARC64)
>> +    !defined(ARM) && !defined(ARM64) && !defined(MIPS) && !defined(MIPS64) && \
>> +    !defined(SPARC64)
>>   #ifdef __alpha__
>>   #define ALPHA
>>   #endif
>> @@ -104,7 +105,11 @@
>>   #define ARM64
>>   #endif
>>   #ifdef __mipsel__
>> +#ifndef __mips64
>>   #define MIPS
>> +#else
>> +#define MIPS64
>> +#endif
>>   #endif
>>   #ifdef __sparc_v9__
>>   #define SPARC64
>> @@ -144,6 +149,9 @@
>>   #ifdef MIPS
>>   #define NR_CPUS  (32)
>>   #endif
>> +#ifdef MIPS64
>> +#define NR_CPUS  (256)
>> +#endif
>>   #ifdef SPARC64
>>   #define NR_CPUS  (4096)
>>   #endif
>> @@ -3343,7 +3351,6 @@ struct arm64_stackframe {
>>
>>   #ifdef MIPS
>>   #define _32BIT_
>> -#define MACHINE_TYPE		"MIPS"
>>
>>   #define PAGEBASE(X)		(((ulong)(X)) & (ulong)machdep->pagemask)
>>
>> @@ -3375,6 +3382,44 @@ struct arm64_stackframe {
>>   #define _MAX_PHYSMEM_BITS	32
>>   #endif  /* MIPS */
>>
>> +#ifdef MIPS64
>> +#define _64BIT_
>> +
>> +#define PAGEBASE(X)		(((ulong)(X)) & (ulong)machdep->pagemask)
>> +#define IS_CKPHYS(X)		(((X) >= 0xffffffff80000000lu) && \
>> +				((X) < 0xffffffffc0000000lu))
>> +#define IS_XKPHYS(X)		(((X) >= 0x8000000000000000lu) && \
>> +				((X) < 0xc000000000000000lu))
>> +
>> +#define PTOV(X) 		((ulong)(X) + 0x9800000000000000lu)
>> +#define VTOP(X) 		(IS_CKPHYS(X) ? ((ulong)(X) & 0x000000001ffffffflu) \
>> +				: ((ulong)(X) & 0x0000fffffffffffflu))
>> +
>> +#define IS_VMALLOC_ADDR(X) (vt->vmalloc_start && (ulong)(X) >= vt->vmalloc_start && !IS_CKPHYS(X))
>> +
>> +#define DEFAULT_MODULES_VADDR   0xffffffffc0000000lu
>> +#define MODULES_VADDR           (machdep->machspec->modules_vaddr)
>> +#define MODULES_END             (machdep->machspec->modules_end)
>> +#define VMALLOC_START           (machdep->machspec->vmalloc_start_addr)
>> +#define VMALLOC_END             (machdep->machspec->vmalloc_end)
>> +
>> +#define __SWP_TYPE_SHIFT        16
>> +#define __SWP_TYPE_BITS         8
>> +#define __SWP_TYPE_MASK         ((1 << __SWP_TYPE_BITS) - 1)
>> +#define __SWP_OFFSET_SHIFT      (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
>> +
>> +#define SWP_TYPE(entry)         (((entry) >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
>> +#define SWP_OFFSET(entry)       ((entry) >> __SWP_OFFSET_SHIFT)
>> +
>> +#define __swp_type(entry)       SWP_TYPE(entry)
>> +#define __swp_offset(entry)     SWP_OFFSET(entry)
>> +
>> +#define TIF_SIGPENDING          (2)
>> +
>> +#define _SECTION_SIZE_BITS      28
>> +#define _MAX_PHYSMEM_BITS       48
>> +#endif  /* MIPS64 */
>> +
>>   #ifdef X86
>>   #define _32BIT_
>>   #define MACHINE_TYPE       "X86"
>> @@ -4415,6 +4460,10 @@ struct machine_specific {
>>   #define MAX_HEXADDR_STRLEN (8)
>>   #define UVADDR_PRLEN       (8)
>>   #endif
>> +#ifdef MIPS64
>> +#define MAX_HEXADDR_STRLEN (16)
>> +#define UVADDR_PRLEN       (16)
>> +#endif
>>   #ifdef SPARC64
>>   #define MAX_HEXADDR_STRLEN (16)
>>   #define UVADDR_PRLEN      (16)
>> @@ -4991,6 +5040,9 @@ void dump_build_data(void);
>>   #ifdef MIPS
>>   #define machdep_init(X) mips_init(X)
>>   #endif
>> +#ifdef MIPS64
>> +#define machdep_init(X) mips64_init(X)
>> +#endif
>>   #ifdef SPARC64
>>   #define machdep_init(X) sparc64_init(X)
>>   #endif
>> @@ -5471,6 +5523,9 @@ void display_help_screen(char *);
>>   #ifdef MIPS
>>   #define dump_machdep_table(X) mips_dump_machdep_table(X)
>>   #endif
>> +#ifdef MIPS64
>> +#define dump_machdep_table(X) mips64_dump_machdep_table(X)
>> +#endif
>>   #ifdef SPARC64
>>   #define dump_machdep_table(X) sparc64_dump_machdep_table(X)
>>   #endif
>> @@ -6348,6 +6403,10 @@ void s390x_dump_machdep_table(ulong);
>>   #define KSYMS_START (0x1)
>>   #endif
>>
>> +#ifdef __mips__ /* MIPS & MIPS64 */
>> +#define MACHINE_TYPE	"MIPS"
>> +#endif
>> +
>>   /*
>>    * mips.c
>>    */
>> @@ -6418,6 +6477,86 @@ struct machine_specific {
>>   #endif /* MIPS */
>>
>>   /*
>> + * mips64.c
>> + */
>> +void mips64_display_regs_from_elf_notes(int, FILE *);
>> +
>> +#ifdef MIPS64
>> +void mips64_init(int);
>> +void mips64_dump_machdep_table(ulong);
>> +
>> +#define display_idt_table() \
>> +	error(FATAL, "-d option is not applicable to MIPS64 architecture\n")
>> +
>> +/* from arch/mips/include/uapi/asm/ptrace.h */
>> +struct mips64_register {
>> +	ulong regs[45];
>> +};
>> +
>> +struct mips64_pt_regs_main {
>> +	ulong regs[32];
>> +	ulong cp0_status;
>> +	ulong hi;
>> +	ulong lo;
>> +};
>> +
>> +struct mips64_pt_regs_cp0 {
>> +	ulong cp0_badvaddr;
>> +	ulong cp0_cause;
>> +	ulong cp0_epc;
>> +};
>> +
>> +struct mips64_unwind_frame {
>> +	unsigned long sp;
>> +	unsigned long pc;
>> +	unsigned long ra;
>> +};
>> +
>> +#define KSYMS_START	(0x1)
>> +
>> +struct machine_specific {
>> +	ulong phys_base;
>> +	ulong vmalloc_start_addr;
>> +	ulong modules_vaddr;
>> +	ulong modules_end;
>> +
>> +	ulong _page_present;
>> +	ulong _page_read;
>> +	ulong _page_write;
>> +	ulong _page_accessed;
>> +	ulong _page_modified;
>> +	ulong _page_huge;
>> +	ulong _page_special;
>> +	ulong _page_protnone;
>> +	ulong _page_global;
>> +	ulong _page_valid;
>> +	ulong _page_no_read;
>> +	ulong _page_no_exec;
>> +	ulong _page_dirty;
>> +
>> +	ulong _pfn_shift;
>> +
>> +	struct mips64_register *crash_task_regs;
>> +};
>> +/* from arch/mips/include/asm/pgtable-bits.h */
>> +#define _PAGE_PRESENT	(machdep->machspec->_page_present)
>> +#define _PAGE_READ	(machdep->machspec->_page_read)
>> +#define _PAGE_WRITE	(machdep->machspec->_page_write)
>> +#define _PAGE_ACCESSED	(machdep->machspec->_page_accessed)
>> +#define _PAGE_MODIFIED	(machdep->machspec->_page_modified)
>> +#define _PAGE_HUGE	(machdep->machspec->_page_huge)
>> +#define _PAGE_SPECIAL	(machdep->machspec->_page_special)
>> +#define _PAGE_PROTNONE	(machdep->machspec->_page_protnone)
>> +#define _PAGE_GLOBAL	(machdep->machspec->_page_global)
>> +#define _PAGE_VALID	(machdep->machspec->_page_valid)
>> +#define _PAGE_NO_READ	(machdep->machspec->_page_no_read)
>> +#define _PAGE_NO_EXEC	(machdep->machspec->_page_no_exec)
>> +#define _PAGE_DIRTY	(machdep->machspec->_page_dirty)
>> +#define _PFN_SHIFT	(machdep->machspec->_pfn_shift)
>> +
>> +#endif /* MIPS64 */
>> +
>> +/*
>>    * sparc64.c
>>    */
>>   #ifdef SPARC64
>> diff --git a/diskdump.c b/diskdump.c
>> index 4f14596..4ad2c50 100644
>> --- a/diskdump.c
>> +++ b/diskdump.c
>> @@ -594,6 +594,9 @@ restart:
>>   	else if (STRNEQ(header->utsname.machine, "mips") &&
>>   	    machine_type_mismatch(file, "MIPS", NULL, 0))
>>   		goto err;
>> +	else if (STRNEQ(header->utsname.machine, "mips64") &&
>> +	    machine_type_mismatch(file, "MIPS", NULL, 0))
>> +		goto err;
> Why do you make MACHINE_TYPE the same as the MIPS one?
> With this, doesn't a MIPS64 crash match a MIPS vmcore?
The value of the machine type e_machine in mips32 or mips64 is MIPS, which
corresponds to EM_MIPS.
The definition in gdb-7.6/include/elf/common.h:110 is as follows:
     #define EM_MIPS        8        /* MIPS R3000 */
But there is no related definition of EM_MIPS64 or other mips64, so both
mips32 and mips64 should use EM_MIPS, and the corresponding e_machine is 
MIPS.

If the MACHINE_TYPE of mips64 is defined as MIPS64, as follows:
     define MACHINE_TYPE     "MIPS64"

The following error will appear when running crash:
     WARNING: machine type mismatch:
               crash utility: MIPS64
               vmcore: MIPS

# readelf -h vmcore
...
   Type:                              CORE (Core file)
   Machine:                           MIPS R3000
...

Therefore, the MACHINE_TYPE of mips32 and mips64 both define MIPS.

Thanks,
Youling
>
> Thanks,
> Kazu
>
>>   	else if (STRNEQ(header->utsname.machine, "s390x") &&
>>   	    machine_type_mismatch(file, "S390X", NULL, 0))
>>   		goto err;
>> diff --git a/lkcd_vmdump_v2_v3.h b/lkcd_vmdump_v2_v3.h
>> index 8d5eae4..984c2c2 100644
>> --- a/lkcd_vmdump_v2_v3.h
>> +++ b/lkcd_vmdump_v2_v3.h
>> @@ -36,7 +36,8 @@
>>   #endif
>>
>>   #if defined(ARM) || defined(X86) || defined(PPC) || defined(S390) || \
>> -	defined(S390X) || defined(ARM64) || defined(MIPS) || defined(SPARC64)
>> +	defined(S390X) || defined(ARM64) || defined(MIPS) || \
>> +	defined(MIPS64) || defined(SPARC64)
>>
>>   /*
>>    * Kernel header file for Linux crash dumps.
>> diff --git a/mips64.c b/mips64.c
>> new file mode 100644
>> index 0000000..5ebe61d
>> --- /dev/null
>> +++ b/mips64.c
>> @@ -0,0 +1,45 @@
>> +/* mips64.c - core analysis suite
>> + *
>> + * Copyright (C) 2021 Loongson Technology Co., Ltd.
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License as published by
>> + * the Free Software Foundation; either version 2 of the License, or
>> + * (at your option) any later version.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + */
>> +#ifdef MIPS64
>> +
>> +#include <elf.h>
>> +#include "defs.h"
>> +
>> +void
>> +mips64_dump_machdep_table(ulong arg)
>> +{
>> +}
>> +
>> +void
>> +mips64_init(int when)
>> +{
>> +}
>> +
>> +void
>> +mips64_display_regs_from_elf_notes(int cpu, FILE *ofp)
>> +{
>> +}
>> +
>> +#else /* !MIPS64 */
>> +
>> +#include "defs.h"
>> +
>> +void
>> +mips64_display_regs_from_elf_notes(int cpu, FILE *ofp)
>> +{
>> +	return;
>> +}
>> +
>> +#endif /* !MIPS64 */
>> diff --git a/symbols.c b/symbols.c
>> index ed5f731..57cb177 100644
>> --- a/symbols.c
>> +++ b/symbols.c
>> @@ -3636,6 +3636,11 @@ is_kernel(char *file)
>>   				goto bailout;
>>   			break;
>>
>> +		case EM_MIPS:
>> +			if (machine_type_mismatch(file, "MIPS", NULL, 0))
>> +				goto bailout;
>> +			break;
>> +
>>   		default:
>>   			if (machine_type_mismatch(file, "(unknown)", NULL, 0))
>>   				goto bailout;
>> @@ -3890,6 +3895,11 @@ is_shared_object(char *file)
>>   			if (machine_type("SPARC64"))
>>   				return TRUE;
>>   			break;
>> +
>> +		case EM_MIPS:
>> +			if (machine_type("MIPS"))
>> +				return TRUE;
>> +			break;
>>   		}
>>
>>   		if (CRASHDEBUG(1))
>> --
>> 2.1.0




More information about the Crash-utility mailing list